private static void TestProcessFile(string fileName, Configurator cfg)
{
// get header from file, build dictionary of index locations for
// columns of interest for this specific log file
Dictionary <string, int> ColumnReference = Util.GetColumnReferences(cfg, fileName);
string line = "";
int filteredRecords = 0;
int counter = 0;
List <string> WorkSet = new List <string>();
int tempMaxRecs = 20;
int spelunk = 735;
using (StreamReader sr = new StreamReader(fileName))
{
line = sr.ReadLine(); // skip the header Row
while ((line = sr.ReadLine()) != null && counter < tempMaxRecs)
{
WorkSet.Add(line);
if ((counter % cfg.BatchSize) == 0)
{
ProcessWorkSet(WorkSet, ColumnReference, cfg);
WorkSet.Clear();
}
counter++;
}
}
// catch remaining records in WorkSet
ProcessWorkSet(WorkSet, ColumnReference, cfg);
WorkSet.Clear();
Console.WriteLine($"{filteredRecords} records filtered from {fileName}");
}
public static void GenerateFile(Configurator cfg)
{
int counter = 0;
string line = "";
var colRef = Util.GetAllColumnReferences(cfg, cfg.MemEffFile);
using (StreamReader sr = new StreamReader(cfg.MemEffFile))
{
line = sr.ReadLine(); // skip header row
while ((line = sr.ReadLine()) != null)
{
var thisRec = line.Split(cfg.Delimiter);
// need to flatten multistep JobID's: if JobID contains "_n", simply remove it
int locationOfUnderscore = thisRec[colRef["JobID"]].IndexOf("_");
if (locationOfUnderscore != -1)
{
thisRec[colRef["JobID"]] = thisRec[colRef["JobID"]].Substring(0, locationOfUnderscore);
}
if (!UserStats.ContainsKey(thisRec[colRef["User"]]))
{
var temp = new SummaryStatsModel
{
Id = thisRec[colRef["User"]],
Account = thisRec[colRef["Account"]],
JobIDFirst = thisRec[colRef["JobID"]],
JobIDLast = thisRec[colRef["JobID"]]
};
UserStats.Add(thisRec[colRef["User"]], temp);
}
if (!AccountStats.ContainsKey(thisRec[colRef["Account"]]))
{
var temp = new SummaryStatsModel
{
Id = thisRec[colRef["Account"]],
JobIDFirst = thisRec[colRef["JobID"]],
JobIDLast = thisRec[colRef["JobID"]]
};
AccountStats.Add(thisRec[colRef["Account"]], temp);
}
int GBHoursReq = int.Parse(thisRec[colRef["GBHoursRequested"]]);
int GBHoursUsed = int.Parse(thisRec[colRef["GBHoursUsed"]]);
double hrs = Util.ConvertTimeStringToHours(thisRec[colRef["Elapsed"]]);
int CoreHours = (int)(hrs * int.Parse(thisRec[colRef["NCPUS"]]));
long thisRecJobID = long.Parse(thisRec[colRef["JobID"]]);
// used for debugging:
//if (!long.TryParse(UserStats[thisRec[colRef["User"]]].JobIDFirst, out jobIDFirst))
// Console.WriteLine($"failed long.TryParse: {thisRec[colRef["JobID"]]}");
//if (!long.TryParse(UserStats[thisRec[colRef["User"]]].JobIDLast, out jobIDLast))
// Console.WriteLine($"failed long.TryParse: {UserStats[thisRec[colRef["User"]]].JobIDLast}");
if (thisRecJobID < long.Parse(UserStats[thisRec[colRef["User"]]].JobIDFirst))
{
UserStats[thisRec[colRef["User"]]].JobIDFirst = thisRecJobID.ToString();
}
if (thisRecJobID > long.Parse(UserStats[thisRec[colRef["User"]]].JobIDLast))
{
UserStats[thisRec[colRef["User"]]].JobIDLast = thisRecJobID.ToString();
}
UserStats[thisRec[colRef["User"]]].JobCount += 1;
UserStats[thisRec[colRef["User"]]].GBHoursReqTotal += GBHoursReq;
UserStats[thisRec[colRef["User"]]].GBHoursUsedTotal += GBHoursUsed;
UserStats[thisRec[colRef["User"]]].CoreHoursTotal += CoreHours;
AccountStats[thisRec[colRef["Account"]]].JobCount += 1;
AccountStats[thisRec[colRef["Account"]]].GBHoursReqTotal += GBHoursReq;
AccountStats[thisRec[colRef["Account"]]].GBHoursUsedTotal += GBHoursUsed;
AccountStats[thisRec[colRef["Account"]]].CoreHoursTotal += CoreHours;
if (thisRecJobID < long.Parse(AccountStats[thisRec[colRef["Account"]]].JobIDFirst))
{
AccountStats[thisRec[colRef["Account"]]].JobIDFirst = thisRecJobID.ToString();
}
if (thisRecJobID > long.Parse(AccountStats[thisRec[colRef["Account"]]].JobIDLast))
{
AccountStats[thisRec[colRef["Account"]]].JobIDLast = thisRecJobID.ToString();
}
counter++;
if (counter % 1000 == 0)
{
Console.Write($"\r Rows Processed: {String.Format("{0:n0}", counter)}");
}
}
Console.Write($"\r Rows Processed: {String.Format("{0:n0}", counter)}{Environment.NewLine}");
}
WriteOutput(cfg, "User", UserStats);
WriteOutput(cfg, "Account", AccountStats);
}
private static void WriteOutput(Configurator cfg, string UserOrAccount, Dictionary <string, SummaryStatsModel> d)
{
StringBuilder sb = new StringBuilder();
sb.Append(UserOrAccount);
sb.Append(cfg.Delimiter);
// hack to bring forward the account to which user is associated
if (UserOrAccount == "User")
{
sb.Append("Account");
sb.Append(cfg.Delimiter);
}
sb.Append("JobCount");
sb.Append(cfg.Delimiter);
sb.Append("AvgCoreHoursPerJob");
sb.Append(cfg.Delimiter);
sb.Append("CoreHours");
sb.Append(cfg.Delimiter);
sb.Append("GBHoursAllocated");
sb.Append(cfg.Delimiter);
sb.Append("GBHoursUsed");
sb.Append(cfg.Delimiter);
sb.Append("PercentGBHoursUsed");
sb.Append(cfg.Delimiter);
sb.Append("JobIDFirst");
sb.Append(cfg.Delimiter);
sb.Append("JobIDLast");
sb.Append(Environment.NewLine);
foreach (var item in d)
{
sb.Append(item.Key);
sb.Append(cfg.Delimiter);
// hack to bring forward the account to which user is associated
if (UserOrAccount == "User")
{
sb.Append(item.Value.Account);
sb.Append(cfg.Delimiter);
}
sb.Append(item.Value.JobCount);
sb.Append(cfg.Delimiter);
double AvgCoreHrsPerJob = item.Value.CoreHoursTotal / item.Value.JobCount;
sb.Append(Math.Round(AvgCoreHrsPerJob, 1));
sb.Append(cfg.Delimiter);
sb.Append(item.Value.CoreHoursTotal);
sb.Append(cfg.Delimiter);
sb.Append(item.Value.GBHoursReqTotal);
sb.Append(cfg.Delimiter);
sb.Append(item.Value.GBHoursUsedTotal);
sb.Append(cfg.Delimiter);
double pct = 0.0;
if (item.Value.GBHoursReqTotal > 0)
{
pct = (item.Value.GBHoursUsedTotal / (double)item.Value.GBHoursReqTotal) * 100.0;
}
sb.Append(Math.Round(pct, 1));
sb.Append(cfg.Delimiter);
sb.Append(item.Value.JobIDFirst);
sb.Append(cfg.Delimiter);
sb.Append(item.Value.JobIDLast);
sb.Append(Environment.NewLine);
}
string outFile = Path.Combine(cfg.OutputLocation, $"summaryStats-{UserOrAccount}.txt");
File.WriteAllText(outFile, sb.ToString());
Console.WriteLine($"File written: {outFile}");
sb.Clear();
}
private static bool FilterOutThisRecord(string line, Dictionary <string, int> colRef, Configurator cfg)
{
// All logic that would cause one to ignore a particular line from the job log data
var thisRec = line.Split(cfg.Delimiter);
// filter unwanted job states
if (thisRec[colRef["State"]] == "RUNNING" ||
thisRec[colRef["State"]] == "SUSPENDED")
{
return(true);
}
return(false);
}
private static void ProcessWorkSet(List <string> workset, Dictionary <string, int> colRef, Configurator cfg)
{
ConcurrentBag <string> cb = new ConcurrentBag <string>();
Parallel.ForEach((workset), (line) =>
{
StringBuilder sb = new StringBuilder();
string[] thisRec = line.Split(cfg.Delimiter);
// straight copy for the fieldsOfInterest
foreach (var item in cfg.ColsOfInterest)
{
sb.Append($"{thisRec[colRef[item]]}");
sb.Append($"{cfg.Delimiter}");
}
// CPUEffeciency: Compare "TotalCPU" with computed field core-walltime; core-walltime = (NCPUS * "Elapsed")
int ncpus = int.Parse(thisRec[colRef["NCPUS"]]);
double hrsElapsed = Util.ConvertTimeStringToHours(thisRec[colRef["Elapsed"]]);
double hrsCoresElapsed = hrsElapsed * ncpus;
string corewalltime = Util.ConvertHoursToTimeString(hrsCoresElapsed);
double CpuEff = 0; // ComputeCPUEffeciency(thisRec[colRef["TotalCPU"]], corewalltime);
sb.Append($"{Math.Round(CpuEff, 2)}");
sb.Append($"{cfg.Delimiter}");
//// MemEffeciency: (MaxRSS / ReqMem) * 100; ReqMem field inlcudes unit, must be stripped and possibly converted
string str_ReqMem = thisRec[colRef["ReqMem"]];
int ReqMem = int.Parse(str_ReqMem.Substring(0, (str_ReqMem.Length - 2)));
string ReqMemUnits = str_ReqMem.Substring(str_ReqMem.Length - 2);
if (ReqMemUnits == "Mc")
{
ReqMem *= int.Parse(thisRec[colRef["NCPUS"]]);
}
float MaxRSS = float.Parse(thisRec[colRef["MaxRSS"]]);
double MemEff = (double)Math.Round(((MaxRSS / ReqMem) * 100), 2);
sb.Append($"{MemEff}");
sb.Append($"{cfg.Delimiter}");
// compute GBHours Requested = (ReqMem / 1000) * Elapsed-Hrs
double GBHoursReq = (ReqMem / 1000.0) * hrsElapsed;
sb.Append($"{(int)(GBHoursReq)}");
sb.Append($"{cfg.Delimiter}");
JobSize js = Util.JobSizeBin(ncpus, ReqMem, (int)hrsElapsed);
var hrsPending = GetPendTime(thisRec[colRef["Submit"]], thisRec[colRef["Start"]]);
cb.Add(sb.ToString());
sb.Clear();
});
File.AppendAllLines(cfg.OutputFile, cb);
}
private static bool FilterOutThisRecord(string line, Dictionary <string, int> colRef, Configurator cfg)
{
// All logic that would cause one to ignore a particular line from the job log data
var thisRec = line.Split(cfg.Delimiter);
int ReqMemThreshold = 3096;
// filter unwanted job states
if (thisRec[colRef["State"]] == "RUNNING" ||
thisRec[colRef["State"]] == "SUSPENDED" ||
thisRec[colRef["State"]] == "OUT_OF_MEMORY")
{
return(true);
}
// filter out partitions for which we do not have a max ReqMem configured
if (!cfg.PartitionMaxReqMem.ContainsKey(thisRec[colRef["Partition"]]))
{
return(true);
}
// filter out records where elapsed time is < 5 minutes
string elapsed = thisRec[colRef["Elapsed"]];
if (elapsed.IndexOf("-") == -1)
{
// if no "-" found within string, then elapsed time was less than one day
string[] hhmmss = elapsed.Split(":");
int hh = int.Parse(hhmmss[0]);
int mm = int.Parse(hhmmss[1]);
if (hh == 0)
{
if (mm < 5)
{
return(true);
}
}
}
// filter out records where requested memory < ReqMemThreshold
string str_ReqMem = thisRec[colRef["ReqMem"]];
if (str_ReqMem.Length < 3) // some cancelled jobs show as "0n" for ReqMem
{
return(true);
}
try
{
int ReqMem = int.Parse(str_ReqMem.Substring(0, (str_ReqMem.Length - 2)));
string ReqMemUnits = str_ReqMem.Substring(str_ReqMem.Length - 2);
if (ReqMemUnits == "Mn")
{
// ReqMem is expressed as memory per node. #nodes always = 1 on longleaf
if (ReqMem < ReqMemThreshold)
{
return(true);
}
}
else if (ReqMemUnits == "Mc")
{
// ReqMem is expressed as memory per core; must multiply by number of cores
int NCPUS = int.Parse(thisRec[colRef["NCPUS"]]);
if ((ReqMem * NCPUS) < ReqMemThreshold)
{
return(true);
}
}
else
{
Console.WriteLine($"*** unrecognized units for ReqMem. str_ReqMem = {str_ReqMem}");
return(true);
}
}
catch (Exception)
{
Console.WriteLine($"*** error parsing ReqMem. str_ReqMem = {str_ReqMem}");
return(true);
}
return(false);
}
