/// <summary>
/// Collect data from client
/// </summary>
/// <param name="client"></param>
public void Collect(PveClient client)
{
var stopwatch = Stopwatch.StartNew();
Version = new Version(1, 0, 0);
Date = DateTime.Now;
Pools = client.Pools.Index().ToEnumerable();
Permissions.Users = client.Access.Users.Index().ToEnumerable();
Permissions.Groups = client.Access.Groups.Index().ToEnumerable();
Permissions.Roles = client.Access.Roles.Index().ToEnumerable();
Permissions.Acl = client.Access.Acl.ReadAcl().ToEnumerable();
Permissions.Domains = client.Access.Domains.Index().ToEnumerable();
//client.Cluster.Ceph.Flags
Storage = client.Storage.Index().ToEnumerable();
Replication = client.Cluster.Replication.Index().ToEnumerable();
Config.Totem = client.Cluster.Config.Totem.Totem().Response.data;
Config.Nodes = client.Cluster.Config.Nodes.Nodes().ToEnumerable();
Backups = client.Cluster.Backup.Index().ToEnumerable();
Resources = client.Cluster.Resources.Resources().ToEnumerable();
Status = client.Cluster.Status.GetStatus().ToEnumerable();
Options = client.Cluster.Options.GetOptions().Response.data;
var status = client.Cluster.Ha.Status.Current.Status();
HA.Status = status.IsSuccessStatusCode
? client.Cluster.Ha.Status.Current.Status().ToEnumerable()
: new List <dynamic>();
HA.Resources = client.Cluster.Ha.Resources.Index().ToEnumerable();
HA.Groups = client.Cluster.Ha.Groups.Index().ToEnumerable();
Firewall.Aliases = client.Cluster.Firewall.Aliases.GetAliases().ToEnumerable();
Firewall.Groups = client.Cluster.Firewall.Groups.ListSecurityGroups().ToEnumerable();
Firewall.Ipset = client.Cluster.Firewall.Ipset.IpsetIndex().ToEnumerable();
Firewall.Macros = client.Cluster.Firewall.Macros.GetMacros().ToEnumerable();
Firewall.Options = client.Cluster.Firewall.Options.GetOptions().Response.data;
Firewall.Refs = client.Cluster.Firewall.Refs.Refs().ToEnumerable();
Firewall.Rules = client.Cluster.Firewall.Rules.GetRules().ToEnumerable();
var nodeList = new List <Node>();
foreach (var nodeItem in client.Nodes.Index().ToEnumerable())
{
var node = client.Nodes[nodeItem.node as string];
var nodeDetail = new Node();
// { Item = nodeItem, };
//nodeDetail.Item.ssl_fingerprint = "REMOVED FOR SECURITY";
var nodeResource = Resources.Where(a => a.type == "node" && a.node == nodeItem.node).FirstOrDefault();
nodeResource.Detail = nodeDetail;
if (nodeItem.status != "online")
{
continue;
}
//nodeResult.Report = node.Report.Report().Response.data;
nodeDetail.PackageVersions = node.Apt.Versions.Versions().ToEnumerable();
nodeDetail.AptUpdate = node.Apt.Update.ListUpdates().ToEnumerable();
nodeDetail.Status = node.Status.Status().Response.data;
nodeDetail.Services = node.Services.Index().ToEnumerable();
nodeDetail.Subscription = node.Subscription.Get().Response.data;
nodeDetail.Subscription.key = "REMOVED FOR SECURITY";
nodeDetail.Firewall.Options = node.Firewall.Options.GetOptions().Response.data;
nodeDetail.Firewall.Rules = node.Firewall.Rules.GetRules().ToEnumerable();
nodeDetail.Version = node.Version.Version().Response.data;
nodeDetail.Certificates = node.Certificates.Info.Info().ToEnumerable();
foreach (var item in nodeDetail.Certificates)
{
item.fingerprint = "REMOVED FOR SECURITY";
item.pem = "REMOVED FOR SECURITY";
}
//ceph
nodeDetail.Ceph.Status = node.Ceph.Status.Status().Response.data;
if (nodeDetail.Ceph.Status != null)
{
nodeDetail.Ceph.Config = node.Ceph.Config.Config().Response.data;
nodeDetail.Ceph.Crush = node.Ceph.Crush.Crush().Response.data;
//nodeDetail.Ceph.Disks = node.Ceph.Disks.Disks().ToEnumerable();
//nodeDetail.Ceph.Flags = node.Ceph.Flags.GetFlags().Response.data;
nodeDetail.Ceph.Mon = node.Ceph.Mon.Listmon().ToEnumerable();
nodeDetail.Ceph.Osd = node.Ceph.Osd.Index().Response.data;
nodeDetail.Ceph.Fs = node.Ceph.Fs.Index().Response.data;
nodeDetail.Ceph.Pools = node.Ceph.Pools.Lspools().ToEnumerable();
nodeDetail.Ceph.Rules = node.Ceph.Rules.Rules().ToEnumerable();
nodeDetail.Ceph.Mds = node.Ceph.Mds.Index().ToEnumerableOrDefault();
nodeDetail.Ceph.Mgr = node.Ceph.Mgr.Index().ToEnumerableOrDefault();
}
//last 2 days
nodeDetail.Tasks = node.Tasks.NodeTasks(errors: true, limit: 1000)
.ToEnumerable()
.Where(a => a.starttime >= DateTimeUnixHelper.ConvertToUnixTime(DateTime.Now.AddDays(-2)));
nodeDetail.Replication = node.Replication.Status().ToEnumerable();
nodeDetail.Hardware.Pci = node.Hardware.Pci.Pciscan().ToEnumerable();
var diskList = node.Disks.List.List().ToEnumerable();
foreach (var item in diskList)
{
item.Smart = node.Disks.Smart.Smart(item.devpath).Response.data;
}
nodeDetail.Disks.List = diskList;
nodeDetail.Disks.Directory = node.Disks.Directory.Index().ToEnumerable();
nodeDetail.Disks.Lvm = node.Disks.Lvm.Index().Response.data;
nodeDetail.Disks.Lvmthin = node.Disks.Lvmthin.Index().ToEnumerable();
var diskZfs = node.Disks.Zfs.Index().ToEnumerable();
foreach (var item in diskZfs)
{
item.Detail = node.Disks.Zfs[item.name].Detail().Response.data;
}
nodeDetail.Disks.Zfs = diskZfs;
nodeDetail.Dns = node.Dns.Dns().Response.data;
nodeDetail.Hosts = node.Hosts.GetEtcHosts().Response.data.data;
nodeDetail.Netstat = node.Netstat.Netstat().ToEnumerable();
nodeDetail.Network = node.Network.Index().ToEnumerable();
nodeDetail.Timezone = node.Time.Time().Response.data.timezone as string;
ReadRrdData(nodeDetail.RrdData, node.Rrddata);
//get storage for backups
var storageWithBackups = node.Storage.Index(format: true, content: "backup")
.ToEnumerable()
.Where(a => a.active == 1)
.Select(a => a.storage as string);
void SetCommonVM(Vm vmDetail, dynamic vm, int vmId, string type)
{
vmDetail.Config.description = "REMOVED FOR SECURITY";
ReadRrdData(vmDetail.RrdData, vm.Rrddata);
vmDetail.Replication = node.Replication.Status(vmId).ToEnumerable();
vmDetail.Tasks = node.Tasks.NodeTasks(errors: true, limit: 1000, vmid: vmId)
.ToEnumerable()
.Where(a => a.starttime >= DateTimeUnixHelper.ConvertToUnixTime(DateTime.Now.AddDays(-2)));
vmDetail.Backups = storageWithBackups.SelectMany(a => node.Storage[a].Content.Index("backup", vmId)
.ToEnumerable())
.ToList();
vmDetail.Permissions = Permissions.Acl.Where(a => a.path == $"/vms/{vmId}").ToList();
//firewall
vmDetail.Firewall.Aliases = ((Result)vm.Firewall.Aliases.GetAliases()).ToEnumerable();
vmDetail.Firewall.Ipset = ((Result)vm.Firewall.Ipset.IpsetIndex()).ToEnumerable();
vmDetail.Firewall.Options = vm.Firewall.Options.GetOptions().Response.data;
vmDetail.Firewall.Refs = ((Result)vm.Firewall.Refs.Refs()).ToEnumerable();
vmDetail.Firewall.Refs = ((Result)vm.Firewall.Rules.GetRules()).ToEnumerable();
var vmResource = Resources.Where(a => a.type == type && a.vmid == vmId).FirstOrDefault();
vmResource.Detail = vmDetail;
}
//lxc
foreach (var vmItem in node.Lxc.Vmlist().ToEnumerable())
{
int vmId = Convert.ToInt32(vmItem.vmid);
var vm = node.Lxc[vmId];
var vmDetail = new Vm
{
Config = vm.Config.VmConfig().Response.data,
Snapshots = vm.Snapshot.List().ToEnumerable(),
Status = vm.Status.Current.VmStatus().Response.data
};
SetCommonVM(vmDetail, vm, vmId, "lxc");
}
//qemu
foreach (var vmItem in node.Qemu.Vmlist().ToEnumerable())
{
int vmId = Convert.ToInt32(vmItem.vmid);
var vm = node.Qemu[vmId];
var vmDetail = new Vm
{
Config = vm.Config.VmConfig().Response.data,
AgentGuestRunning = vm.Agent.Network_Get_Interfaces.Network_Get_Interfaces().Response.data != null,
Snapshots = vm.Snapshot.SnapshotList().ToEnumerable(),
Status = vm.Status.Current.VmStatus().Response.data
};
SetCommonVM(vmDetail, vm, vmId, "qemu");
}
//storages
foreach (var storageItem in node.Storage.Index().ToEnumerable()
.Where(a => a.enabled == 1 && a.active == 1))
{
var storageNode = node.Storage[storageItem.storage as string];
var storeageDetail = new Storage
{
Content = storageNode.Content.Index().Response.data,
Status = storageNode.Status.ReadStatus().Response.data,
};
ReadRrdData(storeageDetail.RrdData, storageNode.Rrddata);
storeageDetail.Permissions = Permissions.Acl.Where(a => a.path == $"/storage/{storageItem.storage}")
.ToList();
var storageResource = Resources.Where(a => a.type == "storage" &&
a.node == nodeItem.node &&
a.storage == storageItem.storage)
.FirstOrDefault();
storageResource.Detail = storeageDetail;
}
nodeList.Add(nodeDetail);
}
stopwatch.Stop();
CollectExecution = stopwatch.Elapsed;
}
private static void CheckNode(ClusterInfo clusterInfo,
List <DiagnosticResult> result,
IEnumerable <dynamic> validResource,
Settings settings)
{
var nodes = validResource.Where(a => a.type == "node" && a.status == "online");
var hasCluster = clusterInfo.Config.Nodes.Count() > 0;
//node
foreach (var node in validResource.Where(a => a.type == "node"))
{
if (node.status != "online")
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Node not online",
Context = DiagnosticResultContext.Node,
SubContext = "Status",
Gravity = DiagnosticResultGravity.Warning,
});
continue;
}
//end of life
var ondOfLife = new Dictionary <string, DateTime>()
{
{ "5", new DateTime(2020, 07, 01) },
{ "4", new DateTime(2018, 06, 01) },
};
DateTime eolDate;
if (ondOfLife.TryGetValue(node.Detail.Version.version.Value.Split('.')[0], out eolDate))
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = $"Version {node.Detail.Version.version.Value} end of life {eolDate.Date} ",
Context = DiagnosticResultContext.Node,
SubContext = "EOL",
Gravity = DiagnosticResultGravity.Warning,
});
}
//subscription
if (node.Detail.Subscription.status.Value != "Active")
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Node not have subscription active",
Context = DiagnosticResultContext.Node,
SubContext = "Subscription",
Gravity = DiagnosticResultGravity.Warning,
});
}
//rdd
CheckNodeRrd(result, settings, node.node.Value, node.Detail.RrdData);
var checkInNodes = new bool[] { true, true, true, true };
foreach (var otherNode in nodes)
{
//version
if (checkInNodes[0] &&
$"{node.release}{node.version}{node.repoid}" !=
$"{otherNode.release}{otherNode.version}{otherNode.repoid}")
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Nodes version not equal",
Context = DiagnosticResultContext.Node,
SubContext = "Version",
Gravity = DiagnosticResultGravity.Critical,
});
checkInNodes[0] = false;
}
//hosts files
if (checkInNodes[1] && node.Detail.Hosts != otherNode.Detail.Hosts)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Nodes hosts configuration not equal",
Context = DiagnosticResultContext.Node,
SubContext = "Hosts",
Gravity = DiagnosticResultGravity.Warning,
});
checkInNodes[1] = false;
}
//DNS
if (checkInNodes[2] &&
(node.Detail.Dns.search != otherNode.Detail.Dns.search ||
node.Detail.Dns.dns1 != otherNode.Detail.Dns.dns1 ||
node.Detail.Dns.dns2 != otherNode.Detail.Dns.dns2))
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Nodes DNS not equal",
Context = DiagnosticResultContext.Node,
SubContext = "DNS",
Gravity = DiagnosticResultGravity.Warning,
});
checkInNodes[2] = false;
}
//timezone
if (checkInNodes[3] && node.Detail.Timezone != otherNode.Detail.Timezone)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Nodes Timezone not equal",
Context = DiagnosticResultContext.Node,
SubContext = "Timezone",
Gravity = DiagnosticResultGravity.Warning,
});
checkInNodes[3] = false;
}
}
//network card
result.AddRange(((IEnumerable <dynamic>)node.Detail.Network)
.Where(a => a.type == "eth" && a.active != 1)
.Select(a => new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0002",
Description = $"Network card '{a.iface}' not active",
Context = DiagnosticResultContext.Node,
SubContext = "Network",
Gravity = DiagnosticResultGravity.Warning,
}));
//Package Versions
var addErrPackageVersions = false;
foreach (var otherNode in nodes)
{
if (addErrPackageVersions)
{
break;
}
foreach (var pkg in node.Detail.PackageVersions)
{
if (!((IEnumerable <dynamic>)node.Detail.PackageVersions)
.Any(a => a.Version == pkg.Version &&
a.Title == pkg.Title &&
a.Package == pkg.Package))
{
addErrPackageVersions = true;
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0001",
Description = "Nodes package version not equal",
Context = DiagnosticResultContext.Node,
SubContext = "PackageVersions",
Gravity = DiagnosticResultGravity.Critical,
});
break;
}
}
}
//Services
var serviceExcluded = new List <string>();
if (!hasCluster)
{
serviceExcluded.Add("corosync");
}
result.AddRange(((IEnumerable <dynamic>)node.Detail.Services)
.Where(a => a.state != "running" && !serviceExcluded.Contains(a.name.Value))
.Select(a => new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0002",
Description = $"Service '{a.desc}' not running",
Context = DiagnosticResultContext.Node,
SubContext = "Service",
Gravity = DiagnosticResultGravity.Warning,
}));
//certificates
result.AddRange(((IEnumerable <dynamic>)node.Detail.Certificates)
.Where(a => DateTimeUnixHelper.UnixTimeToDateTime((long)a.notafter) < clusterInfo.Date)
.Select(a => new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0002",
Description = $"Certificate '{a.filename}' expired",
Context = DiagnosticResultContext.Node,
SubContext = "Certificates",
Gravity = DiagnosticResultGravity.Critical,
}));
//Ceph
if (node.Detail.Ceph.Config != null)
{
//Cluster
if (node.Detail.Ceph.Status.health.status.Value != "HEALTH_OK")
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "IN0001",
Description = $"Ceph status '{node.Detail.Ceph.Status.health.status.Value}'",
Context = DiagnosticResultContext.Node,
SubContext = "Ceph",
Gravity = node.Detail.Ceph.Status.health.status.Value == "HEALTH_WARN" ?
DiagnosticResultGravity.Critical :
DiagnosticResultGravity.Warning
});
}
//PGs active+clean
if (((IEnumerable <dynamic>)node.Detail.Ceph.Status.pgmap.pgs_by_state)
.Where(a => a.state_name == "active+clean").Count() == 0)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "IN0001",
Description = "Ceph PGs not active+clean",
Context = DiagnosticResultContext.Node,
SubContext = "Ceph",
Gravity = DiagnosticResultGravity.Warning
});
}
//Osd
void OsdRic(IEnumerable <dynamic> children)
{
result.AddRange(children.Where(a => a.type == "osd" && a.status != "up" && a.host == node.node)
.Select(a => new DiagnosticResult
{
Id = node.node,
ErrorCode = "WN0002",
Description = $"Osd '{a.name}' not up",
Context = DiagnosticResultContext.Node,
SubContext = "Ceph",
Gravity = DiagnosticResultGravity.Critical,
}));
foreach (var item in children)
{
if (item.children != null)
{
OsdRic(item.children);
}
}
}
OsdRic(node.Detail.Ceph.Osd.root.children);
}
CheckNodeDisk(result, settings, node);
//update
var updateCount = ((IEnumerable <dynamic>)node.Detail.AptUpdate).Count();
if (updateCount > 0)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "IN0001",
Description = $"{updateCount} Update availble",
Context = DiagnosticResultContext.Node,
SubContext = "Update",
Gravity = DiagnosticResultGravity.Info,
});
}
//update important
var updateImportantCount = ((IEnumerable <dynamic>)node.Detail.AptUpdate)
.Where(a => a.Priority == "important").Count();
if (updateImportantCount > 0)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "IN0001",
Description = $"{updateImportantCount} Update Important availble",
Context = DiagnosticResultContext.Node,
SubContext = "Update",
Gravity = DiagnosticResultGravity.Warning,
});
}
//task history
CheckTaskHistory(result,
(IEnumerable <dynamic>)node.Detail.Tasks,
DiagnosticResultContext.Node,
node.node.Value);
//replication
var replCount = ((IEnumerable <dynamic>)node.Detail.Replication)
.Where(a => a.error != null)
.Count();
if (replCount > 0)
{
result.Add(new DiagnosticResult
{
Id = node.node,
ErrorCode = "IN0001",
Description = $"{replCount} Replication has errors",
Context = DiagnosticResultContext.Node,
SubContext = "Replication",
Gravity = DiagnosticResultGravity.Critical,
});
}
}
}
