/// <summary>Shows the first few keys of a directory</summary> public static async Task Show(string[] path, IVarTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct) { int count = 20; if (extras.Count > 0) { int x = extras.Get <int>(0); if (x > 0) { count = x; } } if (path == null || path.Length == 0) { Program.Error(log, "Cannot directory list the content of Root Partition."); return; } // look if there is something under there var folder = await db.Directory.TryOpenAsync(db, path, ct : ct); if (folder != null) { if (folder.Layer == FdbDirectoryPartition.LayerId) { Program.Error(log, "Cannot list the content of a Directory Partition!"); return; } Program.Comment(log, $"# Content of {FdbKey.Dump(folder.GetPrefix())} [{folder.GetPrefix().ToHexaString(' ')}]"); var keys = await db.QueryAsync((tr) => { var query = tr.GetRange(folder.Keys.ToRange()); return(reverse ? query.Reverse().Take(count) : query.Take(count + 1)); }, ct : ct); if (keys.Count > 0) { if (reverse) { keys.Reverse(); } foreach (var key in keys.Take(count)) { log.WriteLine($"...{FdbKey.Dump(folder.ExtractKey(key.Key))} = {key.Value:V}"); } if (!reverse && keys.Count == count + 1) { log.WriteLine("... more"); } } else { log.WriteLine("Folder is empty"); } } }
/// <summary>Shows the first few keys of a directory</summary> public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct) { int count = 20; if (extras.Count > 0) { int x = extras.Get <int>(0); if (x > 0) { count = x; } } // look if there is something under there var folder = await db.Directory.TryOpenAsync(path, cancellationToken : ct); if (folder != null) { log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); var keys = await db.QueryAsync((tr) => { var query = tr.GetRange(folder.ToRange()); return(reverse ? query.Reverse().Take(count) : query.Take(count + 1)); }, cancellationToken : ct); if (keys.Count > 0) { if (reverse) { keys.Reverse(); } foreach (var key in keys.Take(count)) { log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString()); } if (!reverse && keys.Count == count + 1) { log.WriteLine("... more"); } } else { log.WriteLine(" no content found"); } } }
/// <summary>Find the DCs, machines and processes in the cluster</summary> public static async Task Topology(string[] path, IVarTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var coords = await Fdb.System.GetCoordinatorsAsync(db, ct); log.WriteLine($"[Cluster] {coords.Id}"); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(KeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { // Offsets Size Type Name Description // 0 2 Word Version? 0100 (1.0 ?) // 2 4 DWord ??? 0x00 0x20 0xA2 0x00 // 6 2 Word FDBMagic 0xDB 0x0F "FDB" // 8 16 Guid NodeId Unique Process ID // 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 56 4 ??? ?? 4 x 0 // 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below // ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1 // ...4 4 DWord Port 94 11 00 00 -> 4500 // ...8 4 DWord ?? randomish, changes every reboot // ..12 4 DWord ?? randomish, changes every reboot // ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40... // ..20 4 DWord ?? randmoish, changes every reboot ProcessId = kvp.Value.Substring(8, 16).ToHexaString(), MachineId = kvp.Value.Substring(24, 16).ToHexaString(), DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(), Parts = Enumerable.Range(0, 12).Select(i => { int p = 60 + 24 * i; return(new { Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()), Port = kvp.Value.Substring(p + 4, 4).ToInt32(), Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(), Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(), Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(), Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(), }); }).ToList(), Raw = kvp.Value, }), ct ); var numNodes = servers.Select(s => s.ProcessId).Distinct().Count(); var numMachines = servers.Select(s => s.MachineId).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count(); var dcs = servers.GroupBy(x => x.DataCenterId).ToArray(); for (int dcIndex = 0; dcIndex < dcs.Length; dcIndex++) { var dc = dcs[dcIndex]; bool lastDc = dcIndex == dcs.Length - 1; string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key; log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex); var machines = dc.GroupBy(x => x.MachineId).ToArray(); string dcPrefix = lastDc ? " " : "| "; for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++) { var machine = machines[machineIndex]; var lastMachine = machineIndex == machines.Length - 1; string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key; log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId); var procs = machine.ToArray(); string machinePrefix = dcPrefix + (lastMachine ? " " : "| "); for (int procIndex = 0; procIndex < procs.Length; procIndex++) { var proc = procs[procIndex]; bool lastProc = procIndex == procs.Length - 1; log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId); //foreach (var part in proc.Parts) //{ // log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4); //} } } } log.WriteLine(); Program.StdOut(log, $"Found {numNodes:N0} process(es) on {numMachines:N0} machine(s) in {numDCs:N0} datacenter(s)", ConsoleColor.White); log.WriteLine(); }
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct) { // estimate the number of machines... Console.WriteLine("# Detecting cluster topology..."); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { Node = kvp.Value.Substring(8, 16).ToHexaString(), Machine = kvp.Value.Substring(24, 16).ToHexaString(), DataCenter = kvp.Value.Substring(40, 16).ToHexaString() }), ct ); var numNodes = servers.Select(s => s.Node).Distinct().Count(); var numMachines = servers.Select(s => s.Machine).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenter).Distinct().Count(); Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)"); Console.WriteLine("# Reading list of shards..."); // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("# > Found " + ranges.Count + " shards:"); // take a sample var rnd = new Random(1234); int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1); if (sz > 500) { sz = 500; //SAFETY } if (sz < 50) { sz = Math.Max(sz, Math.Min(50, ranges.Count)); } var samples = new List <FdbKeyRange>(); for (int i = 0; i < sz; i++) { int p = rnd.Next(ranges.Count); samples.Add(ranges[p]); ranges.RemoveAt(p); } Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ..."); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total"); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long total = 0; int workers = Math.Min(numMachines, 8); var sw = Stopwatch.StartNew(); var tasks = new List <Task>(); while (samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async() => { var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) { break; } count += data.Count; foreach (var kvp in data.Chunk) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count)); } if (!data.HasMore) { break; } beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref total, totalSize); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold: 2)); } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz)); }
/// <summary>Find the DCs, machines and processes in the cluster</summary> public static async Task Topology(string[] path, IFdbTuple extras, IFdbDatabase db, TextWriter log, CancellationToken ct) { var coords = await Fdb.System.GetCoordinatorsAsync(db, ct); log.WriteLine("[Cluster] {0}", coords.Id); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { // Offsets Size Type Name Description // 0 2 Word Version? 0100 (1.0 ?) // 2 4 DWord ??? 0x00 0x20 0xA2 0x00 // 6 2 Word FDBMagic 0xDB 0x0F "FDB" // 8 16 Guid NodeId Unique Process ID // 24 16 Guid Machine "machine_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 40 16 Guid DataCenter "datacenter_id" field in foundationdb.conf (ends with 8x0 if manually specified) // 56 4 ??? ?? 4 x 0 // 60 12 x24 ARRAY[] ?? array of 12x the same 24-byte struct defined below // ...0 4 DWord IPAddress 01 00 00 7F => 127.0.0.1 // ...4 4 DWord Port 94 11 00 00 -> 4500 // ...8 4 DWord ?? randomish, changes every reboot // ..12 4 DWord ?? randomish, changes every reboot // ..16 4 DWord Size? small L-E integer, usually between 0x20 and 0x40... // ..20 4 DWord ?? randmoish, changes every reboot ProcessId = kvp.Value.Substring(8, 16).ToHexaString(), MachineId = kvp.Value.Substring(24, 16).ToHexaString(), DataCenterId = kvp.Value.Substring(40, 16).ToHexaString(), Parts = Enumerable.Range(0, 12).Select(i => { int p = 60 + 24 * i; return new { Address = new IPAddress(kvp.Value.Substring(p, 4).GetBytes().Reverse().ToArray()), Port = kvp.Value.Substring(p + 4, 4).ToInt32(), Unknown1 = kvp.Value.Substring(p + 8, 4).ToInt32(), Unknown2 = kvp.Value.Substring(p + 12, 4).ToInt32(), Unknown3 = kvp.Value.Substring(p + 16, 4).ToInt32(), Unknown4 = kvp.Value.Substring(p + 20, 4).ToInt32(), }; }).ToList(), Raw = kvp.Value, }), ct ); var numNodes = servers.Select(s => s.ProcessId).Distinct().Count(); var numMachines = servers.Select(s => s.MachineId).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenterId).Distinct().Count(); var dcs = servers.GroupBy(x => x.DataCenterId).ToArray(); for (int dcIndex = 0; dcIndex < dcs.Length;dcIndex++) { var dc = dcs[dcIndex]; bool lastDc = dcIndex == dcs.Length - 1; string dcId = dc.Key.EndsWith("0000000000000000") ? dc.Key.Substring(0, 16) : dc.Key; log.WriteLine((lastDc ? "`- " : "|- ") + "[DataCenter] {0} (#{1})", dcId, dcIndex); var machines = dc.GroupBy(x => x.MachineId).ToArray(); string dcPrefix = lastDc ? " " : "| "; for (int machineIndex = 0; machineIndex < machines.Length; machineIndex++) { var machine = machines[machineIndex]; var lastMachine = machineIndex == machines.Length - 1; string machineId = machine.Key.EndsWith("0000000000000000") ? machine.Key.Substring(0, 16) : machine.Key; log.WriteLine(dcPrefix + (lastMachine ? "`- " : "|- ") + "[Machine] {0}, {1}", machine.First().Parts[0].Address, machineId); var procs = machine.ToArray(); string machinePrefix = dcPrefix + (lastMachine ? " " : "| "); for (int procIndex = 0; procIndex < procs.Length; procIndex++) { var proc = procs[procIndex]; bool lastProc = procIndex == procs.Length - 1; log.WriteLine(machinePrefix + (lastProc ? "`- " : "|- ") + "[Process] {0}:{1}, {2}", proc.Parts[0].Address, proc.Parts[0].Port, proc.ProcessId); //foreach (var part in proc.Parts) //{ // log.WriteLine(machinePrefix + "| -> {0}, {1}, {2:X8}, {3:X8}, {4}, {5:X8}", part.Address, part.Port, part.Unknown1, part.Unknown2, part.Unknown3, part.Unknown4); //} } } } log.WriteLine(); log.WriteLine("Found {0} process(es) on {1} machine(s) in {2} datacenter(s)", numNodes, numMachines, numDCs); log.WriteLine(); }
/// <summary>Shows the first few keys of a directory</summary> public static async Task Show(string[] path, IFdbTuple extras, bool reverse, IFdbDatabase db, TextWriter log, CancellationToken ct) { int count = 20; if (extras.Count > 0) { int x = extras.Get<int>(0); if (x > 0) count = x; } // look if there is something under there var folder = await db.Directory.TryOpenAsync(path, cancellationToken: ct); if (folder != null) { log.WriteLine("# Content of {0} [{1}]", FdbKey.Dump(folder.Key), folder.Key.ToHexaString(' ')); var keys = await db.QueryAsync((tr) => { var query = tr.GetRange(folder.ToRange()); return reverse ? query.Reverse().Take(count) : query.Take(count + 1); }, cancellationToken: ct); if (keys.Count > 0) { if (reverse) keys.Reverse(); foreach (var key in keys.Take(count)) { log.WriteLine("...{0} = {1}", FdbKey.Dump(folder.Extract(key.Key)), key.Value.ToAsciiOrHexaString()); } if (!reverse && keys.Count == count + 1) { log.WriteLine("... more"); } } else { log.WriteLine(" no content found"); } } }
public async Task Run(IFdbDatabase db, TextWriter log, CancellationToken ct) { // estimate the number of machines... Console.WriteLine("# Detecting cluster topology..."); var servers = await db.QueryAsync(tr => tr .WithReadAccessToSystemKeys() .GetRange(FdbKeyRange.StartsWith(Fdb.System.ServerList)) .Select(kvp => new { Node = kvp.Value.Substring(8, 16).ToHexaString(), Machine = kvp.Value.Substring(24, 16).ToHexaString(), DataCenter = kvp.Value.Substring(40, 16).ToHexaString() }), ct ); var numNodes = servers.Select(s => s.Node).Distinct().Count(); var numMachines = servers.Select(s => s.Machine).Distinct().Count(); var numDCs = servers.Select(s => s.DataCenter).Distinct().Count(); Console.WriteLine("# > Found " + numNodes + " process(es) on " + numMachines + " machine(s) in " + numDCs + " datacenter(s)"); Console.WriteLine("# Reading list of shards..."); // dump keyServers var ranges = await Fdb.System.GetChunksAsync(db, FdbKey.MinValue, FdbKey.MaxValue, ct); Console.WriteLine("# > Found " + ranges.Count + " shards:"); // take a sample var rnd = new Random(1234); int sz = Math.Max((int)Math.Ceiling(this.Ratio * ranges.Count), 1); if (sz > 500) sz = 500; //SAFETY if (sz < 50) sz = Math.Max(sz, Math.Min(50, ranges.Count)); var samples = new List<FdbKeyRange>(); for (int i = 0; i < sz; i++) { int p = rnd.Next(ranges.Count); samples.Add(ranges[p]); ranges.RemoveAt(p); } Console.WriteLine("# Sampling " + sz + " out of " + ranges.Count + " shards (" + (100.0 * sz / ranges.Count).ToString("N1") + "%) ..."); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : K+V size distribution", "Count", "Keys", "Values", "Total"); var rangeOptions = new FdbRangeOptions { Mode = FdbStreamingMode.WantAll }; samples = samples.OrderBy(x => x.Begin).ToList(); long total = 0; int workers = Math.Min(numMachines, 8); var sw = Stopwatch.StartNew(); var tasks = new List<Task>(); while(samples.Count > 0) { while (tasks.Count < workers && samples.Count > 0) { var range = samples[0]; samples.RemoveAt(0); tasks.Add(Task.Run(async () => { var hh = new RobustHistogram(RobustHistogram.TimeScale.Ticks); #region Method 1: get_range everything... using (var tr = db.BeginTransaction(ct)) { long keySize = 0; long valueSize = 0; long count = 0; int iter = 0; var beginSelector = FdbKeySelector.FirstGreaterOrEqual(range.Begin); var endSelector = FdbKeySelector.FirstGreaterOrEqual(range.End); while (true) { FdbRangeChunk data = default(FdbRangeChunk); FdbException error = null; try { data = await tr.Snapshot.GetRangeAsync( beginSelector, endSelector, rangeOptions, iter ).ConfigureAwait(false); } catch (FdbException e) { error = e; } if (error != null) { await tr.OnErrorAsync(error.Code).ConfigureAwait(false); continue; } if (data.Count == 0) break; count += data.Count; foreach (var kvp in data.Chunk) { keySize += kvp.Key.Count; valueSize += kvp.Value.Count; hh.Add(TimeSpan.FromTicks(kvp.Key.Count + kvp.Value.Count)); } if (!data.HasMore) break; beginSelector = FdbKeySelector.FirstGreaterThan(data.Last.Key); ++iter; } long totalSize = keySize + valueSize; Interlocked.Add(ref total, totalSize); Console.WriteLine("{0,9}{1,10}{2,10}{3,10} : {4}", count.ToString("N0"), FormatSize(keySize), FormatSize(valueSize), FormatSize(totalSize), hh.GetDistribution(begin: 1, end: 10000, fold:2)); } #endregion #region Method 2: estimate the count using key selectors... //long counter = await Fdb.System.EstimateCountAsync(db, range, ct); //Console.WriteLine("COUNT = " + counter.ToString("N0")); #endregion }, ct)); } var done = await Task.WhenAny(tasks); tasks.Remove(done); } await Task.WhenAll(tasks); sw.Stop(); Console.WriteLine("> Sampled " + FormatSize(total) + " (" + total.ToString("N0") + " bytes) in " + sw.Elapsed.TotalSeconds.ToString("N1") + " sec"); Console.WriteLine("> Estimated total size is " + FormatSize(total * ranges.Count / sz)); }