private static async Task <(long max, long total)> CollectRecordsFromNodes(ClusterRecordsCounter crc) { var apiByName = new Dictionary <string, BobApiClient>(); var vdisks = new List <VDisk>(); foreach (var node in configuration.Nodes) { try { var api = new BobApiClient(node.Address); var status = await api.GetStatus(); if (status == null) { logger.LogError($"Node {node.Address} not available"); continue; } apiByName.Add(status?.Name, api); foreach (var vdisk in status?.VDisks) { if (!vdisks.Any(vd => vd.Id == vdisk.Id)) { vdisks.Add(vdisk); } } } catch (Exception e) { logger.LogError($"Error getting info from node {node.Address}, {e.Message}"); } } return(await crc.CountRecords(apiByName, vdisks)); }
public async Task <(long unique, long withReplicas)> CountRecordsInCluster(Uri baseNode) { using var api = new BobApiClient(baseNode); var nodeObj = await api.GetStatus(); if (nodeObj is null) { throw new Exception("Failed to get node status"); } var vdisks = nodeObj.Value.VDisks; var nodes = await api.GetNodes(); if (nodes is null) { throw new Exception("Failed to get nodes information"); } var apiByName = nodes.ToDictionary(n => n.Name, n => { var url = new Uri($"{baseNode.Scheme}://{n.Address}"); var addr = $"{baseNode.Scheme}://{url.Host}:{baseNode.Port}"; logger.LogInformation($"Found cluster node {addr}"); return(new BobApiClient(new Uri(addr))); }); return(await CountRecords(apiByName, vdisks)); }
private async Task <NodeWithDirs> CreateNode(ConnectionInfo info) { var api = new BobApiClient(info.Uri); var status = await api.GetStatus(); if (status == null) { logger.LogWarning($"Failed to get status from {info.Uri}"); return(null); } var diskDirs = new List <DiskDir>(); foreach (var vDisk in status?.VDisks) { var dirs = await api.GetDirectories(vDisk); foreach (var replica in vDisk.Replicas) { if (replica.Node != status?.Name) { continue; } var parentPath = replica.Path; var name = $"path {replica.Path} on node {info.Uri}"; logger.LogInformation($"Reading disk structure from {name}"); if (dirs == null) { logger.LogWarning($"Failed to get disk structure for {name}"); continue; } var baseDir = dirs.FirstOrDefault(re => re.Path.TrimEnd('/') == replica.Path.TrimEnd('/')); if (baseDir.Path != null) { logger.LogInformation($"Found dir for {name}"); var dir = nodeStructureCreator.ParseDisk(replica.Disk, baseDir, info); if (dir != null) { logger.LogInformation($"Successfully read structure of {name}"); diskDirs.Add(dir); } else { logger.LogWarning($"Structure of {name} can't be parsed"); } } else { logger.LogWarning($"Dir for {name} no found"); } } } var alienDir = await api.GetAlienDirectory(); var alien = nodeStructureCreator.ParseAlien(alienDir, info); return(new NodeWithDirs(info, status?.Name, diskDirs, alien)); }
public async Task CopyDataFromReplica(BobApiClient bobApiClient, BobDisk bobDisk) { if (configuration.PathToDiskStatusAnalyzer == null || !File.Exists(configuration.PathToDiskStatusAnalyzer)) { logger.LogInformation($"DiskStatusAnalyzer path ({configuration.PathToDiskStatusAnalyzer}) is invalid, skipping copy"); return; } var status = await bobApiClient.GetStatus(); if (status is null) { logger.LogError($"Failed to get status from {bobApiClient}"); return; } var destName = status?.Name; var diskName = bobDisk.DiskNameInBob; bool IsCurrent(Replica replica) => replica.Node == destName && replica.Disk == diskName; var vdisks = status?.VDisks.Where(vd => vd.Replicas.Any(IsCurrent)); if (!vdisks.Any()) { logger.LogError($"VDisks with replica ({diskName}, {destName}) not found"); return; } foreach (var vdisk in vdisks) { var bobPath = Path.Combine(bobDisk.BobPath.Path, "bob"); await TryCreateDir(bobPath); await TryCreateDir(Path.Combine(bobPath, vdisk.Id.ToString())); foreach (var replica in vdisk.Replicas) { if (replica.Node == destName) { continue; } logger.LogInformation($"Trying to copy {vdisk} from {replica.Node} to {destName}"); try { await PerformCopy(replica.Node, destName, vdisk.Id); logger.LogInformation($"Successfully copied {vdisk} from {replica.Node} to {destName}"); break; } catch (Exception e) { logger.LogError($"Failed to copy {vdisk} from {replica.Node} to {destName}: {e.Message}"); } } } }
private static async Task RemoveOldPartitions() { if (configuration?.Valid != true) { LogError("Bad configuration"); return; } foreach (var node in configuration.Node) { using var api = new BobApiClient(node.Address); var status = await api.GetStatus(); if (status == null) { continue; } foreach (var vDisk in status?.VDisks) { foreach (var replica in vDisk.Replicas) { if (replica.Node == status?.Name) { LogInfo($"Processing replica of vdisk {vDisk.Id} on node {node}"); var partitions = await api.GetPartitions(vDisk); if (partitions == null) { LogError($"Partitions for {vDisk} not found"); } else { LogInfo($"Found {partitions.Count} partitions on {vDisk}"); await DeleteOldPartitions(node.Address, vDisk, partitions); } } } } } }