/// <summary> /// Collect data from client /// </summary> /// <param name="client"></param> public void Collect(PveClient client) { var stopwatch = Stopwatch.StartNew(); Version = new Version(1, 0, 0); Date = DateTime.Now; Pools = client.Pools.Index().ToEnumerable(); Permissions.Users = client.Access.Users.Index().ToEnumerable(); Permissions.Groups = client.Access.Groups.Index().ToEnumerable(); Permissions.Roles = client.Access.Roles.Index().ToEnumerable(); Permissions.Acl = client.Access.Acl.ReadAcl().ToEnumerable(); Permissions.Domains = client.Access.Domains.Index().ToEnumerable(); //client.Cluster.Ceph.Flags Storage = client.Storage.Index().ToEnumerable(); Replication = client.Cluster.Replication.Index().ToEnumerable(); Config.Totem = client.Cluster.Config.Totem.Totem().Response.data; Config.Nodes = client.Cluster.Config.Nodes.Nodes().ToEnumerable(); Backups = client.Cluster.Backup.Index().ToEnumerable(); Resources = client.Cluster.Resources.Resources().ToEnumerable(); Status = client.Cluster.Status.GetStatus().ToEnumerable(); Options = client.Cluster.Options.GetOptions().Response.data; var status = client.Cluster.Ha.Status.Current.Status(); HA.Status = status.IsSuccessStatusCode ? client.Cluster.Ha.Status.Current.Status().ToEnumerable() : new List <dynamic>(); HA.Resources = client.Cluster.Ha.Resources.Index().ToEnumerable(); HA.Groups = client.Cluster.Ha.Groups.Index().ToEnumerable(); Firewall.Aliases = client.Cluster.Firewall.Aliases.GetAliases().ToEnumerable(); Firewall.Groups = client.Cluster.Firewall.Groups.ListSecurityGroups().ToEnumerable(); Firewall.Ipset = client.Cluster.Firewall.Ipset.IpsetIndex().ToEnumerable(); Firewall.Macros = client.Cluster.Firewall.Macros.GetMacros().ToEnumerable(); Firewall.Options = client.Cluster.Firewall.Options.GetOptions().Response.data; Firewall.Refs = client.Cluster.Firewall.Refs.Refs().ToEnumerable(); Firewall.Rules = client.Cluster.Firewall.Rules.GetRules().ToEnumerable(); var nodeList = new List <Node>(); foreach (var nodeItem in client.Nodes.Index().ToEnumerable()) { var node = client.Nodes[nodeItem.node as string]; var nodeDetail = new Node(); // { Item = nodeItem, }; //nodeDetail.Item.ssl_fingerprint = "REMOVED FOR SECURITY"; var nodeResource = Resources.Where(a => a.type == "node" && a.node == nodeItem.node).FirstOrDefault(); nodeResource.Detail = nodeDetail; if (nodeItem.status != "online") { continue; } //nodeResult.Report = node.Report.Report().Response.data; nodeDetail.PackageVersions = node.Apt.Versions.Versions().ToEnumerable(); nodeDetail.AptUpdate = node.Apt.Update.ListUpdates().ToEnumerable(); nodeDetail.Status = node.Status.Status().Response.data; nodeDetail.Services = node.Services.Index().ToEnumerable(); nodeDetail.Subscription = node.Subscription.Get().Response.data; nodeDetail.Subscription.key = "REMOVED FOR SECURITY"; nodeDetail.Firewall.Options = node.Firewall.Options.GetOptions().Response.data; nodeDetail.Firewall.Rules = node.Firewall.Rules.GetRules().ToEnumerable(); nodeDetail.Version = node.Version.Version().Response.data; nodeDetail.Certificates = node.Certificates.Info.Info().ToEnumerable(); foreach (var item in nodeDetail.Certificates) { item.fingerprint = "REMOVED FOR SECURITY"; item.pem = "REMOVED FOR SECURITY"; } //ceph nodeDetail.Ceph.Status = node.Ceph.Status.Status().Response.data; if (nodeDetail.Ceph.Status != null) { nodeDetail.Ceph.Config = node.Ceph.Config.Config().Response.data; nodeDetail.Ceph.Crush = node.Ceph.Crush.Crush().Response.data; //nodeDetail.Ceph.Disks = node.Ceph.Disks.Disks().ToEnumerable(); //nodeDetail.Ceph.Flags = node.Ceph.Flags.GetFlags().Response.data; nodeDetail.Ceph.Mon = node.Ceph.Mon.Listmon().ToEnumerable(); nodeDetail.Ceph.Osd = node.Ceph.Osd.Index().Response.data; nodeDetail.Ceph.Fs = node.Ceph.Fs.Index().Response.data; nodeDetail.Ceph.Pools = node.Ceph.Pools.Lspools().ToEnumerable(); nodeDetail.Ceph.Rules = node.Ceph.Rules.Rules().ToEnumerable(); nodeDetail.Ceph.Mds = node.Ceph.Mds.Index().ToEnumerableOrDefault(); nodeDetail.Ceph.Mgr = node.Ceph.Mgr.Index().ToEnumerableOrDefault(); } //last 2 days nodeDetail.Tasks = node.Tasks.NodeTasks(errors: true, limit: 1000) .ToEnumerable() .Where(a => a.starttime >= DateTimeUnixHelper.ConvertToUnixTime(DateTime.Now.AddDays(-2))); nodeDetail.Replication = node.Replication.Status().ToEnumerable(); nodeDetail.Hardware.Pci = node.Hardware.Pci.Pciscan().ToEnumerable(); var diskList = node.Disks.List.List().ToEnumerable(); foreach (var item in diskList) { item.Smart = node.Disks.Smart.Smart(item.devpath).Response.data; } nodeDetail.Disks.List = diskList; nodeDetail.Disks.Directory = node.Disks.Directory.Index().ToEnumerable(); nodeDetail.Disks.Lvm = node.Disks.Lvm.Index().Response.data; nodeDetail.Disks.Lvmthin = node.Disks.Lvmthin.Index().ToEnumerable(); var diskZfs = node.Disks.Zfs.Index().ToEnumerable(); foreach (var item in diskZfs) { item.Detail = node.Disks.Zfs[item.name].Detail().Response.data; } nodeDetail.Disks.Zfs = diskZfs; nodeDetail.Dns = node.Dns.Dns().Response.data; nodeDetail.Hosts = node.Hosts.GetEtcHosts().Response.data.data; nodeDetail.Netstat = node.Netstat.Netstat().ToEnumerable(); nodeDetail.Network = node.Network.Index().ToEnumerable(); nodeDetail.Timezone = node.Time.Time().Response.data.timezone as string; ReadRrdData(nodeDetail.RrdData, node.Rrddata); //get storage for backups var storageWithBackups = node.Storage.Index(format: true, content: "backup") .ToEnumerable() .Where(a => a.active == 1) .Select(a => a.storage as string); void SetCommonVM(Vm vmDetail, dynamic vm, int vmId, string type) { vmDetail.Config.description = "REMOVED FOR SECURITY"; ReadRrdData(vmDetail.RrdData, vm.Rrddata); vmDetail.Replication = node.Replication.Status(vmId).ToEnumerable(); vmDetail.Tasks = node.Tasks.NodeTasks(errors: true, limit: 1000, vmid: vmId) .ToEnumerable() .Where(a => a.starttime >= DateTimeUnixHelper.ConvertToUnixTime(DateTime.Now.AddDays(-2))); vmDetail.Backups = storageWithBackups.SelectMany(a => node.Storage[a].Content.Index("backup", vmId) .ToEnumerable()) .ToList(); vmDetail.Permissions = Permissions.Acl.Where(a => a.path == $"/vms/{vmId}").ToList(); //firewall vmDetail.Firewall.Aliases = ((Result)vm.Firewall.Aliases.GetAliases()).ToEnumerable(); vmDetail.Firewall.Ipset = ((Result)vm.Firewall.Ipset.IpsetIndex()).ToEnumerable(); vmDetail.Firewall.Options = vm.Firewall.Options.GetOptions().Response.data; vmDetail.Firewall.Refs = ((Result)vm.Firewall.Refs.Refs()).ToEnumerable(); vmDetail.Firewall.Refs = ((Result)vm.Firewall.Rules.GetRules()).ToEnumerable(); var vmResource = Resources.Where(a => a.type == type && a.vmid == vmId).FirstOrDefault(); vmResource.Detail = vmDetail; } //lxc foreach (var vmItem in node.Lxc.Vmlist().ToEnumerable()) { int vmId = Convert.ToInt32(vmItem.vmid); var vm = node.Lxc[vmId]; var vmDetail = new Vm { Config = vm.Config.VmConfig().Response.data, Snapshots = vm.Snapshot.List().ToEnumerable(), Status = vm.Status.Current.VmStatus().Response.data }; SetCommonVM(vmDetail, vm, vmId, "lxc"); } //qemu foreach (var vmItem in node.Qemu.Vmlist().ToEnumerable()) { int vmId = Convert.ToInt32(vmItem.vmid); var vm = node.Qemu[vmId]; var vmDetail = new Vm { Config = vm.Config.VmConfig().Response.data, AgentGuestRunning = vm.Agent.Network_Get_Interfaces.Network_Get_Interfaces().Response.data != null, Snapshots = vm.Snapshot.SnapshotList().ToEnumerable(), Status = vm.Status.Current.VmStatus().Response.data }; SetCommonVM(vmDetail, vm, vmId, "qemu"); } //storages foreach (var storageItem in node.Storage.Index().ToEnumerable() .Where(a => a.enabled == 1 && a.active == 1)) { var storageNode = node.Storage[storageItem.storage as string]; var storeageDetail = new Storage { Content = storageNode.Content.Index().Response.data, Status = storageNode.Status.ReadStatus().Response.data, }; ReadRrdData(storeageDetail.RrdData, storageNode.Rrddata); storeageDetail.Permissions = Permissions.Acl.Where(a => a.path == $"/storage/{storageItem.storage}") .ToList(); var storageResource = Resources.Where(a => a.type == "storage" && a.node == nodeItem.node && a.storage == storageItem.storage) .FirstOrDefault(); storageResource.Detail = storeageDetail; } nodeList.Add(nodeDetail); } stopwatch.Stop(); CollectExecution = stopwatch.Elapsed; }
private static void CheckNode(ClusterInfo clusterInfo, List <DiagnosticResult> result, IEnumerable <dynamic> validResource, Settings settings) { var nodes = validResource.Where(a => a.type == "node" && a.status == "online"); var hasCluster = clusterInfo.Config.Nodes.Count() > 0; //node foreach (var node in validResource.Where(a => a.type == "node")) { if (node.status != "online") { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Node not online", Context = DiagnosticResultContext.Node, SubContext = "Status", Gravity = DiagnosticResultGravity.Warning, }); continue; } //end of life var ondOfLife = new Dictionary <string, DateTime>() { { "5", new DateTime(2020, 07, 01) }, { "4", new DateTime(2018, 06, 01) }, }; DateTime eolDate; if (ondOfLife.TryGetValue(node.Detail.Version.version.Value.Split('.')[0], out eolDate)) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = $"Version {node.Detail.Version.version.Value} end of life {eolDate.Date} ", Context = DiagnosticResultContext.Node, SubContext = "EOL", Gravity = DiagnosticResultGravity.Warning, }); } //subscription if (node.Detail.Subscription.status.Value != "Active") { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Node not have subscription active", Context = DiagnosticResultContext.Node, SubContext = "Subscription", Gravity = DiagnosticResultGravity.Warning, }); } //rdd CheckNodeRrd(result, settings, node.node.Value, node.Detail.RrdData); var checkInNodes = new bool[] { true, true, true, true }; foreach (var otherNode in nodes) { //version if (checkInNodes[0] && $"{node.release}{node.version}{node.repoid}" != $"{otherNode.release}{otherNode.version}{otherNode.repoid}") { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Nodes version not equal", Context = DiagnosticResultContext.Node, SubContext = "Version", Gravity = DiagnosticResultGravity.Critical, }); checkInNodes[0] = false; } //hosts files if (checkInNodes[1] && node.Detail.Hosts != otherNode.Detail.Hosts) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Nodes hosts configuration not equal", Context = DiagnosticResultContext.Node, SubContext = "Hosts", Gravity = DiagnosticResultGravity.Warning, }); checkInNodes[1] = false; } //DNS if (checkInNodes[2] && (node.Detail.Dns.search != otherNode.Detail.Dns.search || node.Detail.Dns.dns1 != otherNode.Detail.Dns.dns1 || node.Detail.Dns.dns2 != otherNode.Detail.Dns.dns2)) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Nodes DNS not equal", Context = DiagnosticResultContext.Node, SubContext = "DNS", Gravity = DiagnosticResultGravity.Warning, }); checkInNodes[2] = false; } //timezone if (checkInNodes[3] && node.Detail.Timezone != otherNode.Detail.Timezone) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Nodes Timezone not equal", Context = DiagnosticResultContext.Node, SubContext = "Timezone", Gravity = DiagnosticResultGravity.Warning, }); checkInNodes[3] = false; } } //network card result.AddRange(((IEnumerable <dynamic>)node.Detail.Network) .Where(a => a.type == "eth" && a.active != 1) .Select(a => new DiagnosticResult { Id = node.node, ErrorCode = "WN0002", Description = $"Network card '{a.iface}' not active", Context = DiagnosticResultContext.Node, SubContext = "Network", Gravity = DiagnosticResultGravity.Warning, })); //Package Versions var addErrPackageVersions = false; foreach (var otherNode in nodes) { if (addErrPackageVersions) { break; } foreach (var pkg in node.Detail.PackageVersions) { if (!((IEnumerable <dynamic>)node.Detail.PackageVersions) .Any(a => a.Version == pkg.Version && a.Title == pkg.Title && a.Package == pkg.Package)) { addErrPackageVersions = true; result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "WN0001", Description = "Nodes package version not equal", Context = DiagnosticResultContext.Node, SubContext = "PackageVersions", Gravity = DiagnosticResultGravity.Critical, }); break; } } } //Services var serviceExcluded = new List <string>(); if (!hasCluster) { serviceExcluded.Add("corosync"); } result.AddRange(((IEnumerable <dynamic>)node.Detail.Services) .Where(a => a.state != "running" && !serviceExcluded.Contains(a.name.Value)) .Select(a => new DiagnosticResult { Id = node.node, ErrorCode = "WN0002", Description = $"Service '{a.desc}' not running", Context = DiagnosticResultContext.Node, SubContext = "Service", Gravity = DiagnosticResultGravity.Warning, })); //certificates result.AddRange(((IEnumerable <dynamic>)node.Detail.Certificates) .Where(a => DateTimeUnixHelper.UnixTimeToDateTime((long)a.notafter) < clusterInfo.Date) .Select(a => new DiagnosticResult { Id = node.node, ErrorCode = "WN0002", Description = $"Certificate '{a.filename}' expired", Context = DiagnosticResultContext.Node, SubContext = "Certificates", Gravity = DiagnosticResultGravity.Critical, })); //Ceph if (node.Detail.Ceph.Config != null) { //Cluster if (node.Detail.Ceph.Status.health.status.Value != "HEALTH_OK") { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "IN0001", Description = $"Ceph status '{node.Detail.Ceph.Status.health.status.Value}'", Context = DiagnosticResultContext.Node, SubContext = "Ceph", Gravity = node.Detail.Ceph.Status.health.status.Value == "HEALTH_WARN" ? DiagnosticResultGravity.Critical : DiagnosticResultGravity.Warning }); } //PGs active+clean if (((IEnumerable <dynamic>)node.Detail.Ceph.Status.pgmap.pgs_by_state) .Where(a => a.state_name == "active+clean").Count() == 0) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "IN0001", Description = "Ceph PGs not active+clean", Context = DiagnosticResultContext.Node, SubContext = "Ceph", Gravity = DiagnosticResultGravity.Warning }); } //Osd void OsdRic(IEnumerable <dynamic> children) { result.AddRange(children.Where(a => a.type == "osd" && a.status != "up" && a.host == node.node) .Select(a => new DiagnosticResult { Id = node.node, ErrorCode = "WN0002", Description = $"Osd '{a.name}' not up", Context = DiagnosticResultContext.Node, SubContext = "Ceph", Gravity = DiagnosticResultGravity.Critical, })); foreach (var item in children) { if (item.children != null) { OsdRic(item.children); } } } OsdRic(node.Detail.Ceph.Osd.root.children); } CheckNodeDisk(result, settings, node); //update var updateCount = ((IEnumerable <dynamic>)node.Detail.AptUpdate).Count(); if (updateCount > 0) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "IN0001", Description = $"{updateCount} Update availble", Context = DiagnosticResultContext.Node, SubContext = "Update", Gravity = DiagnosticResultGravity.Info, }); } //update important var updateImportantCount = ((IEnumerable <dynamic>)node.Detail.AptUpdate) .Where(a => a.Priority == "important").Count(); if (updateImportantCount > 0) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "IN0001", Description = $"{updateImportantCount} Update Important availble", Context = DiagnosticResultContext.Node, SubContext = "Update", Gravity = DiagnosticResultGravity.Warning, }); } //task history CheckTaskHistory(result, (IEnumerable <dynamic>)node.Detail.Tasks, DiagnosticResultContext.Node, node.node.Value); //replication var replCount = ((IEnumerable <dynamic>)node.Detail.Replication) .Where(a => a.error != null) .Count(); if (replCount > 0) { result.Add(new DiagnosticResult { Id = node.node, ErrorCode = "IN0001", Description = $"{replCount} Replication has errors", Context = DiagnosticResultContext.Node, SubContext = "Replication", Gravity = DiagnosticResultGravity.Critical, }); } } }