static string readSherpa(string filename, bool print)
{
string line;
string host = "new";
int cores = 0;
List<string> summary = new List<string>();
List<string> times = new List<string>();
List<double> cpu_all_usr = new List<double>();
List<double> memsum = new List<double>();
List<string> disklabels = new List<string>();
List<double> disksizes = new List<double>();
List<string> disksizesb = new List<string>();
List<string> netwuts = new List<string>();
List<List<string>> topList = new List<List<string>>();
List<double>[] diskbusy;
List<double>[] netties;
string[] dix;
string datime = "";
string ddate = "";
string ttime = "";
string warnings = "";
using (StreamReader reader = new StreamReader(filename)) {
/*read in each line of text then do stuff with it*/
//small while loop only does maybe 50lines before breaking
while ((line = reader.ReadLine()) != null) {//this is the prelim loop to make the primary loop go quicker
summary.Add(line);
string[] values = line.Split(',');
if (values[1] == "time") {
if (values[0] == "AAA")
ttime = values[2];
datime = String.Join("",values[2].Split(new[] { ':', '.' }));
}
if (values[1] == "date") {
if (values[0] == "AAA")
ddate = values[2];
datime = String.Join("", values[2].Split('-')) + "_"+datime;
}
if (values[1] == "host")
host = values[2];
if (values[1] == "cpus")
cores = Convert.ToInt32(values[2]);
if (values[0] == "NET") {//first line of NET data from the file
foreach (string nets in values.Skip(2)) { //for all the nets presented on this line (skipping the first 2 garbage lines)
if(nets != "") netwuts.Add(nets);//all the things, each iface, each bond, eths, los.. everything from the ifconfig
}
}
if (values[0] == "DISKBUSY") {//first line of DISKBUSY holds disk names
foreach (string diskN in values.Skip(2)) { //for all the disk labels presented on this line (skipping the first 2 garbage lines)
if(diskN != "") disklabels.Add(diskN);//all sd and dm partitions, just keep it all in there
}
}
if (values[0] == "BBBP"){
if (values[2] == "/proc/partitions") {
try {
dix = values[3].Split(new[] { ' ', '\"' }, StringSplitOptions.RemoveEmptyEntries);
if (dix[0] != "major") {
disksizes.Add(Convert.ToDouble(dix[2])/1000);
disksizesb.Add(dix[3]);
}
} catch { }
} else if (values[2] == "/proc/1/stat")
break;
}
}//some background info was gathered from AAA
netties = new List<double>[netwuts.Count()];
for (int i = 0; i < netties.Count(); i++) {
netties[i] = new List<double>();//so many I dont even
}//we now have netwuts.count netties[]s; each netties is a double list we can add each(EVERY SINGLE) line nmon records
diskbusy = new List<double>[disklabels.Count()];
for (int i = 0; i < disklabels.Count(); i++) {
diskbusy[i] = new List<double>();//almost as many I dont even
}//we now have disklabels.count diskbusy[]s; each diskbusy is a double list we can add each(EVERY SINGLE) line nmon records
List<process> processes = new List<process>();
while ((line = reader.ReadLine()) != null) { //Got all the prelim done, now do the rest of the file
string[] values = line.Split(',');
/*switch was faster than an if block*/
try{
switch (values[0]) {
case "ZZZZ":
times.Add(values[2] + " " + values[3]);
break;
case "TOP":
List<string> topstuff = new List<string>();
//TOP,+PID,Time,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command
//TOP,0031885,T0050,92.9,89.2,3.7,1416768,1105388,144,0,143692,34,0,osii_dbms_adapt
topstuff.Add(values[2].Substring(1, values[2].Length - 1));//time in front of topstuff
for (int i = 1; i < values.Count(); i++) {
if (i != 2) {//skip time
topstuff.Add(values[i]);//add each value starting from 1 (skipping 2)
//Time,+PID,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command
//0050,0031885,92.9,89.2,3.7,1416768,1105388,144,0,143692,34,0,osii_dbms_adapt
}
}
topList.Add(topstuff);
string TOPpid = topstuff[1];
int TPID = Convert.ToInt32(TOPpid);
string TOPres = topstuff[6];
string TOPcpu = topstuff[2];
bool found = false;
foreach (process x in processes) {
if (x.pid == TPID) {
x.add(TOPres, TOPcpu);
found = true;
break;
}
}
if (!found) {
process t = new process(TOPpid, TOPres, TOPcpu,topstuff[12]);
processes.Add(t);
}
break;
case "CPU_ALL":
if (values[2] != "User%") {
cpu_all_usr.Add((Convert.ToDouble(values[2]) + Convert.ToDouble(values[3])));
}
break;
case "MEM":
if (values[2] != "memtotal") {
memsum.Add(100.0 * (1 - ((Convert.ToDouble(values[6]) + Convert.ToDouble(values[11]) + Convert.ToDouble(values[14])) / Convert.ToDouble(values[2]))));
}
break;
case "NET":
Parallel.ForEach(values.Skip(2), (nets, y, i) => {
if (nets != "") netties[i].Add(Convert.ToDouble(nets));
});
break;
case "DISKBUSY":
Parallel.ForEach(values.Skip(2), (disk, y, i) => {
diskbusy[i].Add(Convert.ToDouble(disk));
});
break;
//etc
default: //poison buckets barf pile
break;
}//end switch
}catch(Exception e){
string m = e.Message;
}
}//end while
//eqMB = LeastSquares(memList);
//double stdD1 = StandardDev(memList);
//List<double> memList2 = sigma(memList, stdD1, eqMB);
//cX = memList2.Count();
//eqMB = LeastSquares(memList2);
//double stdD2 = StandardDev(memList2)*1.2;
//List<double> memList3 = sigma(memList2, stdD2, eqMB);
//eqMB = LeastSquares(memList3);
//if (eqMB[0] < 0) {
// leaking = true;
double leakthreshold = 0.1;
List<string> procXS = new List<string>();
Parallel.ForEach(processes, x => {
double leaks = x.isLeaking();
if (leaks > leakthreshold && x.total > (times.Count*0.25)) {
procXS.Add(x.pid + " (" + x.command+") rate of: "+Convert.ToDouble(leaks));
}
});
if (procXS.Count > 0) {
warnings += host+" has potential memory leaks in:" + System.Environment.NewLine;
Console.WriteLine("Recommended graph investigation - possible leaks!");
foreach (string l in procXS) {
warnings += l + System.Environment.NewLine;
}
}
List<string> cpuXS = new List<string>();
Parallel.ForEach(processes, x => {
double cpuU = x.isHigh();
if (cpuU>40) {
cpuXS.Add(x.pid + " (" + x.command + ") averaging %"+Convert.ToInt32(cpuU));
}
});
if (cpuXS.Count > 0) {
warnings += host+" has potential high CPU usage in:" + System.Environment.NewLine;
Console.WriteLine("Recommended graph investigation - found high CPU!");
foreach (string l in cpuXS) {
warnings += l + System.Environment.NewLine;
}
}
}//done file handling
//inframortions
//feels like a bad way to do this, but worked well
string dump = "";
dump += host + " from "+ttime+" "+ddate+" time chunks: " + times.Count + System.Environment.NewLine;
//CPU
dump += host + " CPU(%) average: " + cpu_all_usr.Average() + System.Environment.NewLine;
dump += host + " CPU(%) max: " + cpu_all_usr.Max() + System.Environment.NewLine;
//MEM
dump += host+ " MEM(%) average: " + memsum.Average()+System.Environment.NewLine;
dump += host + " MEM(%) max: " + memsum.Max() + System.Environment.NewLine;
//DISKBUSY
for(int i=0;i<disklabels.Count;i++){
if(disklabels[i].Substring(0,1)!="d"){
dump += host + " DISKBUSY(%) avg for " + disklabels[i] + ": " + diskbusy[i].Average() + System.Environment.NewLine;
dump += host + " DISKBUSY(%) max for " + disklabels[i] + ": " + diskbusy[i].Max() + System.Environment.NewLine;
}
}
//DISKBUSY weights
double sdSum = 0.0;
double diskweight = 0.0;
double diskmaxes = 0.0;
for (int i = 0; i < disksizesb.Count; i++) {
if (disksizesb[i].Substring(0, 1) == "s") {
sdSum += disksizes[i];
}
}
for (int i = 0; i < disklabels.Count; i++) {
if (disklabels[i].Substring(0, 1) == "s") {
for (int j = 0; j < disksizesb.Count; j++) {
if (disksizesb[j] == disklabels[i]) {
diskweight += (diskbusy[i].Average() * disksizes[j]);
diskmaxes += (diskbusy[i].Max() * disksizes[j]);
}
}
}
}
dump += host + " weighted DISKBUSY(%) avg: " + diskweight / sdSum + System.Environment.NewLine;
dump += host + " weighted DISKBUSY(%) max: " + diskmaxes / sdSum + System.Environment.NewLine;
//NET
for (int i = 0; i < netwuts.Count; i++) {
if (netwuts[i].Substring(0, 2) != "lo") {//we dont need to see the loopback
dump += host + " NET average for " + netwuts[i] + ": " + netties[i].Average() + System.Environment.NewLine;
}
}
//warnings!!!
if (warnings != "") {
dump += warnings;
}
//CSV file stuff
if (print) {
string topTitle = "Time,PID,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command";
using (StreamWriter file = new StreamWriter(host +"_"+ datime+"_TOP.csv")) {
file.WriteLine(topTitle);
for (int i = 0; i < topList.Count; i++) {
try {
file.WriteLine(times[Convert.ToInt16(topList[i][0])-1] + "," + string.Join(",", topList[i].Skip(1)));//wat, dats right
} catch {
// *shrug do nothing
}
}
}
}
Console.WriteLine("Finishing " + host +" from "+datime);
return (dump);
}
static string readSherpa(string filename, bool print)
{
string line;
string host = "new";
int cores = 0;
List <string> summary = new List <string>();
List <string> times = new List <string>();
List <double> cpu_all_usr = new List <double>();
List <double> memsum = new List <double>();
List <string> disklabels = new List <string>();
List <double> disksizes = new List <double>();
List <string> disksizesb = new List <string>();
List <string> netwuts = new List <string>();
List <List <string> > topList = new List <List <string> >();
List <double>[] diskbusy;
List <double>[] netties;
string[] dix;
string datime = "";
string ddate = "";
string ttime = "";
string warnings = "";
using (StreamReader reader = new StreamReader(filename)) {
/*read in each line of text then do stuff with it*/
//small while loop only does maybe 50lines before breaking
while ((line = reader.ReadLine()) != null)//this is the prelim loop to make the primary loop go quicker
{
summary.Add(line);
string[] values = line.Split(',');
if (values[1] == "time")
{
if (values[0] == "AAA")
{
ttime = values[2];
}
datime = String.Join("", values[2].Split(new[] { ':', '.' }));
}
if (values[1] == "date")
{
if (values[0] == "AAA")
{
ddate = values[2];
}
datime = String.Join("", values[2].Split('-')) + "_" + datime;
}
if (values[1] == "host")
{
host = values[2];
}
if (values[1] == "cpus")
{
cores = Convert.ToInt32(values[2]);
}
if (values[0] == "NET") //first line of NET data from the file
{
foreach (string nets in values.Skip(2)) //for all the nets presented on this line (skipping the first 2 garbage lines)
{
if (nets != "")
{
netwuts.Add(nets);//all the things, each iface, each bond, eths, los.. everything from the ifconfig
}
}
}
if (values[0] == "DISKBUSY") //first line of DISKBUSY holds disk names
{
foreach (string diskN in values.Skip(2)) //for all the disk labels presented on this line (skipping the first 2 garbage lines)
{
if (diskN != "")
{
disklabels.Add(diskN);//all sd and dm partitions, just keep it all in there
}
}
}
if (values[0] == "BBBP")
{
if (values[2] == "/proc/partitions")
{
try {
dix = values[3].Split(new[] { ' ', '\"' }, StringSplitOptions.RemoveEmptyEntries);
if (dix[0] != "major")
{
disksizes.Add(Convert.ToDouble(dix[2]) / 1000);
disksizesb.Add(dix[3]);
}
} catch { }
}
else if (values[2] == "/proc/1/stat")
{
break;
}
}
}//some background info was gathered from AAA
netties = new List <double> [netwuts.Count()];
for (int i = 0; i < netties.Count(); i++)
{
netties[i] = new List <double>();//so many I dont even
}//we now have netwuts.count netties[]s; each netties is a double list we can add each(EVERY SINGLE) line nmon records
diskbusy = new List <double> [disklabels.Count()];
for (int i = 0; i < disklabels.Count(); i++)
{
diskbusy[i] = new List <double>();//almost as many I dont even
}//we now have disklabels.count diskbusy[]s; each diskbusy is a double list we can add each(EVERY SINGLE) line nmon records
List <process> processes = new List <process>();
while ((line = reader.ReadLine()) != null) //Got all the prelim done, now do the rest of the file
{
string[] values = line.Split(',');
/*switch was faster than an if block*/
try{
switch (values[0])
{
case "ZZZZ":
times.Add(values[2] + " " + values[3]);
break;
case "TOP":
List <string> topstuff = new List <string>();
//TOP,+PID,Time,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command
//TOP,0031885,T0050,92.9,89.2,3.7,1416768,1105388,144,0,143692,34,0,osii_dbms_adapt
topstuff.Add(values[2].Substring(1, values[2].Length - 1));//time in front of topstuff
for (int i = 1; i < values.Count(); i++)
{
if (i != 2) //skip time
{
topstuff.Add(values[i]); //add each value starting from 1 (skipping 2)
//Time,+PID,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command
//0050,0031885,92.9,89.2,3.7,1416768,1105388,144,0,143692,34,0,osii_dbms_adapt
}
}
topList.Add(topstuff);
string TOPpid = topstuff[1];
int TPID = Convert.ToInt32(TOPpid);
string TOPres = topstuff[6];
string TOPcpu = topstuff[2];
bool found = false;
foreach (process x in processes)
{
if (x.pid == TPID)
{
x.add(TOPres, TOPcpu);
found = true;
break;
}
}
if (!found)
{
process t = new process(TOPpid, TOPres, TOPcpu, topstuff[12]);
processes.Add(t);
}
break;
case "CPU_ALL":
if (values[2] != "User%")
{
cpu_all_usr.Add((Convert.ToDouble(values[2]) + Convert.ToDouble(values[3])));
}
break;
case "MEM":
if (values[2] != "memtotal")
{
memsum.Add(100.0 * (1 - ((Convert.ToDouble(values[6]) + Convert.ToDouble(values[11]) + Convert.ToDouble(values[14])) / Convert.ToDouble(values[2]))));
}
break;
case "NET":
Parallel.ForEach(values.Skip(2), (nets, y, i) => {
if (nets != "")
{
netties[i].Add(Convert.ToDouble(nets));
}
});
break;
case "DISKBUSY":
Parallel.ForEach(values.Skip(2), (disk, y, i) => {
diskbusy[i].Add(Convert.ToDouble(disk));
});
break;
//etc
default: //poison buckets barf pile
break;
}//end switch
}catch (Exception e) {
string m = e.Message;
}
}//end while
//eqMB = LeastSquares(memList);
//double stdD1 = StandardDev(memList);
//List<double> memList2 = sigma(memList, stdD1, eqMB);
//cX = memList2.Count();
//eqMB = LeastSquares(memList2);
//double stdD2 = StandardDev(memList2)*1.2;
//List<double> memList3 = sigma(memList2, stdD2, eqMB);
//eqMB = LeastSquares(memList3);
//if (eqMB[0] < 0) {
// leaking = true;
double leakthreshold = 0.1;
List <string> procXS = new List <string>();
Parallel.ForEach(processes, x => {
double leaks = x.isLeaking();
if (leaks > leakthreshold && x.total > (times.Count * 0.25))
{
procXS.Add(x.pid + " (" + x.command + ") rate of: " + Convert.ToDouble(leaks));
}
});
if (procXS.Count > 0)
{
warnings += host + " has potential memory leaks in:" + System.Environment.NewLine;
Console.WriteLine("Recommended graph investigation - possible leaks!");
foreach (string l in procXS)
{
warnings += l + System.Environment.NewLine;
}
}
List <string> cpuXS = new List <string>();
Parallel.ForEach(processes, x => {
double cpuU = x.isHigh();
if (cpuU > 40)
{
cpuXS.Add(x.pid + " (" + x.command + ") averaging %" + Convert.ToInt32(cpuU));
}
});
if (cpuXS.Count > 0)
{
warnings += host + " has potential high CPU usage in:" + System.Environment.NewLine;
Console.WriteLine("Recommended graph investigation - found high CPU!");
foreach (string l in cpuXS)
{
warnings += l + System.Environment.NewLine;
}
}
}//done file handling
//inframortions
//feels like a bad way to do this, but worked well
string dump = "";
dump += host + " from " + ttime + " " + ddate + " time chunks: " + times.Count + System.Environment.NewLine;
//CPU
dump += host + " CPU(%) average: " + cpu_all_usr.Average() + System.Environment.NewLine;
dump += host + " CPU(%) max: " + cpu_all_usr.Max() + System.Environment.NewLine;
//MEM
dump += host + " MEM(%) average: " + memsum.Average() + System.Environment.NewLine;
dump += host + " MEM(%) max: " + memsum.Max() + System.Environment.NewLine;
//DISKBUSY
for (int i = 0; i < disklabels.Count; i++)
{
if (disklabels[i].Substring(0, 1) != "d")
{
dump += host + " DISKBUSY(%) avg for " + disklabels[i] + ": " + diskbusy[i].Average() + System.Environment.NewLine;
dump += host + " DISKBUSY(%) max for " + disklabels[i] + ": " + diskbusy[i].Max() + System.Environment.NewLine;
}
}
//DISKBUSY weights
double sdSum = 0.0;
double diskweight = 0.0;
double diskmaxes = 0.0;
for (int i = 0; i < disksizesb.Count; i++)
{
if (disksizesb[i].Substring(0, 1) == "s")
{
sdSum += disksizes[i];
}
}
for (int i = 0; i < disklabels.Count; i++)
{
if (disklabels[i].Substring(0, 1) == "s")
{
for (int j = 0; j < disksizesb.Count; j++)
{
if (disksizesb[j] == disklabels[i])
{
diskweight += (diskbusy[i].Average() * disksizes[j]);
diskmaxes += (diskbusy[i].Max() * disksizes[j]);
}
}
}
}
dump += host + " weighted DISKBUSY(%) avg: " + diskweight / sdSum + System.Environment.NewLine;
dump += host + " weighted DISKBUSY(%) max: " + diskmaxes / sdSum + System.Environment.NewLine;
//NET
for (int i = 0; i < netwuts.Count; i++)
{
if (netwuts[i].Substring(0, 2) != "lo")//we dont need to see the loopback
{
dump += host + " NET average for " + netwuts[i] + ": " + netties[i].Average() + System.Environment.NewLine;
}
}
//warnings!!!
if (warnings != "")
{
dump += warnings;
}
//CSV file stuff
if (print)
{
string topTitle = "Time,PID,%CPU,%Usr,%Sys,Size,ResSet,ResText,ResData,ShdLib,MajorFault,MinorFault,Command";
using (StreamWriter file = new StreamWriter(host + "_" + datime + "_TOP.csv")) {
file.WriteLine(topTitle);
for (int i = 0; i < topList.Count; i++)
{
try {
file.WriteLine(times[Convert.ToInt16(topList[i][0]) - 1] + "," + string.Join(",", topList[i].Skip(1)));//wat, dats right
} catch {
// *shrug do nothing
}
}
}
}
Console.WriteLine("Finishing " + host + " from " + datime);
return(dump);
}
