private static async Task <(long max, long total)> CollectRecordsFromNodes(ClusterRecordsCounter crc)
{
var apiByName = new Dictionary <string, BobApiClient>();
var vdisks = new List <VDisk>();
foreach (var node in configuration.Nodes)
{
try
{
var api = new BobApiClient(node.Address);
var status = await api.GetStatus();
if (status == null)
{
logger.LogError($"Node {node.Address} not available");
continue;
}
apiByName.Add(status?.Name, api);
foreach (var vdisk in status?.VDisks)
{
if (!vdisks.Any(vd => vd.Id == vdisk.Id))
{
vdisks.Add(vdisk);
}
}
}
catch (Exception e)
{
logger.LogError($"Error getting info from node {node.Address}, {e.Message}");
}
}
return(await crc.CountRecords(apiByName, vdisks));
}
public async Task <(long unique, long withReplicas)> CountRecordsInCluster(Uri baseNode)
{
using var api = new BobApiClient(baseNode);
var nodeObj = await api.GetStatus();
if (nodeObj is null)
{
throw new Exception("Failed to get node status");
}
var vdisks = nodeObj.Value.VDisks;
var nodes = await api.GetNodes();
if (nodes is null)
{
throw new Exception("Failed to get nodes information");
}
var apiByName = nodes.ToDictionary(n => n.Name, n =>
{
var url = new Uri($"{baseNode.Scheme}://{n.Address}");
var addr = $"{baseNode.Scheme}://{url.Host}:{baseNode.Port}";
logger.LogInformation($"Found cluster node {addr}");
return(new BobApiClient(new Uri(addr)));
});
return(await CountRecords(apiByName, vdisks));
}
private async Task <NodeWithDirs> CreateNode(ConnectionInfo info)
{
var api = new BobApiClient(info.Uri);
var status = await api.GetStatus();
if (status == null)
{
logger.LogWarning($"Failed to get status from {info.Uri}");
return(null);
}
var diskDirs = new List <DiskDir>();
foreach (var vDisk in status?.VDisks)
{
var dirs = await api.GetDirectories(vDisk);
foreach (var replica in vDisk.Replicas)
{
if (replica.Node != status?.Name)
{
continue;
}
var parentPath = replica.Path;
var name = $"path {replica.Path} on node {info.Uri}";
logger.LogInformation($"Reading disk structure from {name}");
if (dirs == null)
{
logger.LogWarning($"Failed to get disk structure for {name}");
continue;
}
var baseDir = dirs.FirstOrDefault(re => re.Path.TrimEnd('/') == replica.Path.TrimEnd('/'));
if (baseDir.Path != null)
{
logger.LogInformation($"Found dir for {name}");
var dir = nodeStructureCreator.ParseDisk(replica.Disk, baseDir, info);
if (dir != null)
{
logger.LogInformation($"Successfully read structure of {name}");
diskDirs.Add(dir);
}
else
{
logger.LogWarning($"Structure of {name} can't be parsed");
}
}
else
{
logger.LogWarning($"Dir for {name} no found");
}
}
}
var alienDir = await api.GetAlienDirectory();
var alien = nodeStructureCreator.ParseAlien(alienDir, info);
return(new NodeWithDirs(info, status?.Name, diskDirs, alien));
}
public async Task CopyDataFromReplica(BobApiClient bobApiClient, BobDisk bobDisk)
{
if (configuration.PathToDiskStatusAnalyzer == null || !File.Exists(configuration.PathToDiskStatusAnalyzer))
{
logger.LogInformation($"DiskStatusAnalyzer path ({configuration.PathToDiskStatusAnalyzer}) is invalid, skipping copy");
return;
}
var status = await bobApiClient.GetStatus();
if (status is null)
{
logger.LogError($"Failed to get status from {bobApiClient}");
return;
}
var destName = status?.Name;
var diskName = bobDisk.DiskNameInBob;
bool IsCurrent(Replica replica) => replica.Node == destName && replica.Disk == diskName;
var vdisks = status?.VDisks.Where(vd => vd.Replicas.Any(IsCurrent));
if (!vdisks.Any())
{
logger.LogError($"VDisks with replica ({diskName}, {destName}) not found");
return;
}
foreach (var vdisk in vdisks)
{
var bobPath = Path.Combine(bobDisk.BobPath.Path, "bob");
await TryCreateDir(bobPath);
await TryCreateDir(Path.Combine(bobPath, vdisk.Id.ToString()));
foreach (var replica in vdisk.Replicas)
{
if (replica.Node == destName)
{
continue;
}
logger.LogInformation($"Trying to copy {vdisk} from {replica.Node} to {destName}");
try
{
await PerformCopy(replica.Node, destName, vdisk.Id);
logger.LogInformation($"Successfully copied {vdisk} from {replica.Node} to {destName}");
break;
}
catch (Exception e)
{
logger.LogError($"Failed to copy {vdisk} from {replica.Node} to {destName}: {e.Message}");
}
}
}
}
private static async Task RemoveOldPartitions()
{
if (configuration?.Valid != true)
{
LogError("Bad configuration");
return;
}
foreach (var node in configuration.Node)
{
using var api = new BobApiClient(node.Address);
var status = await api.GetStatus();
if (status == null)
{
continue;
}
foreach (var vDisk in status?.VDisks)
{
foreach (var replica in vDisk.Replicas)
{
if (replica.Node == status?.Name)
{
LogInfo($"Processing replica of vdisk {vDisk.Id} on node {node}");
var partitions = await api.GetPartitions(vDisk);
if (partitions == null)
{
LogError($"Partitions for {vDisk} not found");
}
else
{
LogInfo($"Found {partitions.Count} partitions on {vDisk}");
await DeleteOldPartitions(node.Address, vDisk, partitions);
}
}
}
}
}
}