private async Task MountDisks(BobApiClient bobApiClient)
{
var disks = await FindDisks();
foreach (var disk in disks)
{
foreach (var volume in disk.Volumes)
{
if (neededInfoStorage.ShouldBeProcessed(volume))
{
await disksMounter.MountVolume(volume, bobApiClient);
if (!neededInfoStorage.IsProtected(volume))
{
var bobPath = neededInfoStorage.FindBobPath(volume);
if (bobPath != null)
{
await bobPathPreparer.PrepareBobPath(bobPath);
}
else
{
logger.LogInformation($"No bobpath found for {volume}");
}
}
}
}
}
}
private async Task <bool> TryCleanPreviousData(Volume volume, BobApiClient bobApiClient, MountPath path)
{
int count = 0;
await TryStopBobdisk(volume, bobApiClient);
while (count++ < configuration.MaxUmountRetries)
{
try
{
var disks = await disksFinder.FindDisks();
if (disks.Any(d => d.Volumes.Count > 0 && d.Volumes.Any(v => v.MountPath?.Equals(path) == true && v.IsMounted)))
{
logger.LogInformation($"Trying to unmount previous disks in {path}");
await processInvoker.InvokeSudoProcess("umount", path.ToString());
logger.LogInformation($"Successfully umounted previous disks in {path}");
}
return(true);
}
catch (ProcessFailedException e) when(e.ExitCode == 32)
{
await Task.Delay(1000);
}
}
private async Task <List <RestartInfo?> > CopyAliensFromNode(NodeWithDirs node, List <NodeWithDirs> nodes, Configuration config)
{
var result = new List <RestartInfo?>();
var client = new BobApiClient(node.Uri);
var disks = await client.GetDisksToMonitor();
var alienDisk = disks.FirstOrDefault(d => d.Path == node.AlienDir.Path).Name;
if (alienDisk == null)
{
return(result);
}
await client.StopDisk(alienDisk);
await client.StartDisk(alienDisk);
var tasks = new List <Task <RestartInfo?[]> >();
foreach (var alienNode in node.AlienDir.Nodes)
{
tasks.Add(CopyAlien(alienNode, nodes, config));
}
return(await Task.WhenAll(tasks).ContinueWith(t => t.Result.SelectMany(_ => _).ToList()));
}
private static async Task <(long max, long total)> CollectRecordsFromNodes(ClusterRecordsCounter crc)
{
var apiByName = new Dictionary <string, BobApiClient>();
var vdisks = new List <VDisk>();
foreach (var node in configuration.Nodes)
{
try
{
var api = new BobApiClient(node.Address);
var status = await api.GetStatus();
if (status == null)
{
logger.LogError($"Node {node.Address} not available");
continue;
}
apiByName.Add(status?.Name, api);
foreach (var vdisk in status?.VDisks)
{
if (!vdisks.Any(vd => vd.Id == vdisk.Id))
{
vdisks.Add(vdisk);
}
}
}
catch (Exception e)
{
logger.LogError($"Error getting info from node {node.Address}, {e.Message}");
}
}
return(await crc.CountRecords(apiByName, vdisks));
}
public async Task CheckAndUpdate(BobApiClient bobApiClient)
{
await PrepareDisks();
await FormatDisks();
await MountDisks(bobApiClient);
await AlterFSTab();
}
private static async Task <Configuration> GetConfiguration(BobApiClient bobApiClient)
{
logger.LogInformation("Reading configuration...");
var configuration = serviceProvider.GetRequiredService <Configuration>();
try
{
await configuration.ReadFromFile(configFile);
await configuration.AddEntriesFromBob(bobApiClient);
await configuration.SaveToFile(configFile);
return(configuration);
}
catch (Exception e)
{
logger.LogError($"Exception while getting info from config: {e.Message}{Environment.NewLine}{e.StackTrace}");
throw;
}
finally
{
logger.LogInformation("Reading configuration done");
}
}
public async Task <(long unique, long withReplicas)> CountRecordsInCluster(Uri baseNode)
{
using var api = new BobApiClient(baseNode);
var nodeObj = await api.GetStatus();
if (nodeObj is null)
{
throw new Exception("Failed to get node status");
}
var vdisks = nodeObj.Value.VDisks;
var nodes = await api.GetNodes();
if (nodes is null)
{
throw new Exception("Failed to get nodes information");
}
var apiByName = nodes.ToDictionary(n => n.Name, n =>
{
var url = new Uri($"{baseNode.Scheme}://{n.Address}");
var addr = $"{baseNode.Scheme}://{url.Host}:{baseNode.Port}";
logger.LogInformation($"Found cluster node {addr}");
return(new BobApiClient(new Uri(addr)));
});
return(await CountRecords(apiByName, vdisks));
}
private async Task <NodeWithDirs> CreateNode(ConnectionInfo info)
{
var api = new BobApiClient(info.Uri);
var status = await api.GetStatus();
if (status == null)
{
logger.LogWarning($"Failed to get status from {info.Uri}");
return(null);
}
var diskDirs = new List <DiskDir>();
foreach (var vDisk in status?.VDisks)
{
var dirs = await api.GetDirectories(vDisk);
foreach (var replica in vDisk.Replicas)
{
if (replica.Node != status?.Name)
{
continue;
}
var parentPath = replica.Path;
var name = $"path {replica.Path} on node {info.Uri}";
logger.LogInformation($"Reading disk structure from {name}");
if (dirs == null)
{
logger.LogWarning($"Failed to get disk structure for {name}");
continue;
}
var baseDir = dirs.FirstOrDefault(re => re.Path.TrimEnd('/') == replica.Path.TrimEnd('/'));
if (baseDir.Path != null)
{
logger.LogInformation($"Found dir for {name}");
var dir = nodeStructureCreator.ParseDisk(replica.Disk, baseDir, info);
if (dir != null)
{
logger.LogInformation($"Successfully read structure of {name}");
diskDirs.Add(dir);
}
else
{
logger.LogWarning($"Structure of {name} can't be parsed");
}
}
else
{
logger.LogWarning($"Dir for {name} no found");
}
}
}
var alienDir = await api.GetAlienDirectory();
var alien = nodeStructureCreator.ParseAlien(alienDir, info);
return(new NodeWithDirs(info, status?.Name, diskDirs, alien));
}
public async Task CopyDataFromReplica(BobApiClient bobApiClient, BobDisk bobDisk)
{
if (configuration.PathToDiskStatusAnalyzer == null || !File.Exists(configuration.PathToDiskStatusAnalyzer))
{
logger.LogInformation($"DiskStatusAnalyzer path ({configuration.PathToDiskStatusAnalyzer}) is invalid, skipping copy");
return;
}
var status = await bobApiClient.GetStatus();
if (status is null)
{
logger.LogError($"Failed to get status from {bobApiClient}");
return;
}
var destName = status?.Name;
var diskName = bobDisk.DiskNameInBob;
bool IsCurrent(Replica replica) => replica.Node == destName && replica.Disk == diskName;
var vdisks = status?.VDisks.Where(vd => vd.Replicas.Any(IsCurrent));
if (!vdisks.Any())
{
logger.LogError($"VDisks with replica ({diskName}, {destName}) not found");
return;
}
foreach (var vdisk in vdisks)
{
var bobPath = Path.Combine(bobDisk.BobPath.Path, "bob");
await TryCreateDir(bobPath);
await TryCreateDir(Path.Combine(bobPath, vdisk.Id.ToString()));
foreach (var replica in vdisk.Replicas)
{
if (replica.Node == destName)
{
continue;
}
logger.LogInformation($"Trying to copy {vdisk} from {replica.Node} to {destName}");
try
{
await PerformCopy(replica.Node, destName, vdisk.Id);
logger.LogInformation($"Successfully copied {vdisk} from {replica.Node} to {destName}");
break;
}
catch (Exception e)
{
logger.LogError($"Failed to copy {vdisk} from {replica.Node} to {destName}: {e.Message}");
}
}
}
}
public async Task <IEnumerable <BobDisk> > GenerateConfigFromBob(BobApiClient bobApiClient)
{
var disks = await bobApiClient.GetDisksToMonitor();
if (disks == null)
{
logger.LogError($"Failed to get bob disks from {bobApiClient}");
return(Enumerable.Empty <BobDisk>());
}
var physicalDisks = await disksFinder.FindDisks();
var infos = disks.Where(d => d.IsActive).Select(d => FindInfo(d, physicalDisks)).Where(i => i != null);
return(infos);
}
public async Task StartDisks(BobApiClient bobApiClient)
{
var disksToStart = configuration.MonitoringEntries;
var disks = await disksFinder.FindDisks();
foreach (var i in disksToStart)
{
var inactiveDisks = await bobApiClient.GetInactiveDisks();
if (inactiveDisks == null)
{
return;
}
if (!inactiveDisks.Any(d => d.Name == i.DiskNameInBob))
{
continue;
}
var disk = disks.FirstOrDefault(d => d.Volumes.Any(v => v.MountPath.Equals(i.MountPath) && v.IsMounted));
var volume = disk?.Volumes.First(v => v.MountPath.Equals(i.MountPath) && v.IsMounted);
if (disks.Count == 0 ||
!disks.Any(d => !d.NoVolumes && d.Volumes.Any(v => v.MountPath.Equals(i.MountPath) && v.IsMounted && !v.IsReadOnly)))
{
continue;
}
logger.LogInformation($"Trying to start disk {i}");
if (!configuration.KnownUuids.Contains(volume.UUID))
{
await disksCopier.CopyDataFromReplica(bobApiClient, i);
}
configuration.SaveUUID(await disksMonitor.GetUUID(i));
logger.LogInformation($"Starting bobdisk {i}...");
int retry = 0;
while (!await bobApiClient.StartDisk(i.DiskNameInBob) && retry++ < configuration.StartRetryCount)
{
logger.LogWarning($"Failed to start bobdisk in try {retry}, trying again");
}
if (retry == configuration.StartRetryCount)
{
logger.LogError($"Failed to start bobdisk {i}");
}
else
{
logger.LogInformation($"Bobdisk {i} started");
}
}
}
private static async Task DeleteOldPartitions(Uri uri, VDisk vDisk, List <string> partitions)
{
using var api = new BobApiClient(uri);
foreach (var partition in partitions)
{
var partitionObject = await api.GetPartition(vDisk, partition);
if (partitionObject is null)
{
LogError($"Failed to get partition {partition} on {vDisk}");
}
else if (GetDateTimeFromTimestamp(partitionObject?.Timestamp) < configuration.Threshold)
{
await api.DeletePartition(vDisk, partitionObject?.Timestamp);
LogInfo($"Deleted partition {partition} on {vDisk}");
}
}
}
private static async Task RemoveOldPartitions()
{
if (configuration?.Valid != true)
{
LogError("Bad configuration");
return;
}
foreach (var node in configuration.Node)
{
using var api = new BobApiClient(node.Address);
var status = await api.GetStatus();
if (status == null)
{
continue;
}
foreach (var vDisk in status?.VDisks)
{
foreach (var replica in vDisk.Replicas)
{
if (replica.Node == status?.Name)
{
LogInfo($"Processing replica of vdisk {vDisk.Id} on node {node}");
var partitions = await api.GetPartitions(vDisk);
if (partitions == null)
{
LogError($"Partitions for {vDisk} not found");
}
else
{
LogInfo($"Found {partitions.Count} partitions on {vDisk}");
await DeleteOldPartitions(node.Address, vDisk, partitions);
}
}
}
}
}
}
public async Task AddEntriesFromBob(BobApiClient bobApiClient)
{
var disks = await disksFinder.FindDisks();
var infos = await configGenerator.GenerateConfigFromBob(bobApiClient);
foreach (var i in infos)
{
logger.LogInformation($"Received {i} from bob");
var conflict = MonitoringEntries.FirstOrDefault(bd => bd.DiskNameInBob == i.DiskNameInBob && bd.BobPath.Equals(i.BobPath) && !bd.Equals(i));
if (conflict != null)
{
logger.LogWarning($"Conflict: disk from bob {i} conflicts with disk from config {conflict}");
continue;
}
var uuid = disks.SelectMany(d => d.Volumes).FirstOrDefault(v => v.PhysicalId.Equals(i.PhysicalId) && v.IsFormatted)?.UUID;
if (uuid != null)
{
SaveUUID(uuid);
}
MonitoringEntries.Add(i);
}
}
public async Task MountVolume(Volume volume, BobApiClient bobApiClient)
{
if (volume.IsMounted && !volume.IsReadOnly)
{
logger.LogDebug($"{volume} is already mounted");
return;
}
if (!volume.Mountable)
{
logger.LogDebug($"{volume} can't be mounted");
return;
}
var mountPath = neededInfoStorage.FindMountPath(volume);
if (mountPath != null)
{
var path = mountPath?.Path;
if (!Directory.Exists(path))
{
await processInvoker.InvokeSudoProcess("mkdir", path);
}
if (await TryCleanPreviousData(volume, bobApiClient, mountPath.Value) && !volume.IsMounted)
{
logger.LogInformation($"Mounting {volume} to {mountPath}");
await processInvoker.InvokeSudoProcess("mount", volume.DevPath.Path, path);
await processInvoker.SetDirPermissionsAndOwner(path, configuration.MountPointPermissions, configuration.MountPointOwner);
logger.LogInformation($"Successfully mounted {volume} to {mountPath}");
}
}
else
{
logger.LogInformation($"No mount path found for {volume}");
}
}
