private static void TestProcessFile(string fileName, Configurator cfg) { // get header from file, build dictionary of index locations for // columns of interest for this specific log file Dictionary <string, int> ColumnReference = Util.GetColumnReferences(cfg, fileName); string line = ""; int filteredRecords = 0; int counter = 0; List <string> WorkSet = new List <string>(); int tempMaxRecs = 20; int spelunk = 735; using (StreamReader sr = new StreamReader(fileName)) { line = sr.ReadLine(); // skip the header Row while ((line = sr.ReadLine()) != null && counter < tempMaxRecs) { WorkSet.Add(line); if ((counter % cfg.BatchSize) == 0) { ProcessWorkSet(WorkSet, ColumnReference, cfg); WorkSet.Clear(); } counter++; } } // catch remaining records in WorkSet ProcessWorkSet(WorkSet, ColumnReference, cfg); WorkSet.Clear(); Console.WriteLine($"{filteredRecords} records filtered from {fileName}"); }
public static void GenerateFile(Configurator cfg) { int counter = 0; string line = ""; var colRef = Util.GetAllColumnReferences(cfg, cfg.MemEffFile); using (StreamReader sr = new StreamReader(cfg.MemEffFile)) { line = sr.ReadLine(); // skip header row while ((line = sr.ReadLine()) != null) { var thisRec = line.Split(cfg.Delimiter); // need to flatten multistep JobID's: if JobID contains "_n", simply remove it int locationOfUnderscore = thisRec[colRef["JobID"]].IndexOf("_"); if (locationOfUnderscore != -1) { thisRec[colRef["JobID"]] = thisRec[colRef["JobID"]].Substring(0, locationOfUnderscore); } if (!UserStats.ContainsKey(thisRec[colRef["User"]])) { var temp = new SummaryStatsModel { Id = thisRec[colRef["User"]], Account = thisRec[colRef["Account"]], JobIDFirst = thisRec[colRef["JobID"]], JobIDLast = thisRec[colRef["JobID"]] }; UserStats.Add(thisRec[colRef["User"]], temp); } if (!AccountStats.ContainsKey(thisRec[colRef["Account"]])) { var temp = new SummaryStatsModel { Id = thisRec[colRef["Account"]], JobIDFirst = thisRec[colRef["JobID"]], JobIDLast = thisRec[colRef["JobID"]] }; AccountStats.Add(thisRec[colRef["Account"]], temp); } int GBHoursReq = int.Parse(thisRec[colRef["GBHoursRequested"]]); int GBHoursUsed = int.Parse(thisRec[colRef["GBHoursUsed"]]); double hrs = Util.ConvertTimeStringToHours(thisRec[colRef["Elapsed"]]); int CoreHours = (int)(hrs * int.Parse(thisRec[colRef["NCPUS"]])); long thisRecJobID = long.Parse(thisRec[colRef["JobID"]]); // used for debugging: //if (!long.TryParse(UserStats[thisRec[colRef["User"]]].JobIDFirst, out jobIDFirst)) // Console.WriteLine($"failed long.TryParse: {thisRec[colRef["JobID"]]}"); //if (!long.TryParse(UserStats[thisRec[colRef["User"]]].JobIDLast, out jobIDLast)) // Console.WriteLine($"failed long.TryParse: {UserStats[thisRec[colRef["User"]]].JobIDLast}"); if (thisRecJobID < long.Parse(UserStats[thisRec[colRef["User"]]].JobIDFirst)) { UserStats[thisRec[colRef["User"]]].JobIDFirst = thisRecJobID.ToString(); } if (thisRecJobID > long.Parse(UserStats[thisRec[colRef["User"]]].JobIDLast)) { UserStats[thisRec[colRef["User"]]].JobIDLast = thisRecJobID.ToString(); } UserStats[thisRec[colRef["User"]]].JobCount += 1; UserStats[thisRec[colRef["User"]]].GBHoursReqTotal += GBHoursReq; UserStats[thisRec[colRef["User"]]].GBHoursUsedTotal += GBHoursUsed; UserStats[thisRec[colRef["User"]]].CoreHoursTotal += CoreHours; AccountStats[thisRec[colRef["Account"]]].JobCount += 1; AccountStats[thisRec[colRef["Account"]]].GBHoursReqTotal += GBHoursReq; AccountStats[thisRec[colRef["Account"]]].GBHoursUsedTotal += GBHoursUsed; AccountStats[thisRec[colRef["Account"]]].CoreHoursTotal += CoreHours; if (thisRecJobID < long.Parse(AccountStats[thisRec[colRef["Account"]]].JobIDFirst)) { AccountStats[thisRec[colRef["Account"]]].JobIDFirst = thisRecJobID.ToString(); } if (thisRecJobID > long.Parse(AccountStats[thisRec[colRef["Account"]]].JobIDLast)) { AccountStats[thisRec[colRef["Account"]]].JobIDLast = thisRecJobID.ToString(); } counter++; if (counter % 1000 == 0) { Console.Write($"\r Rows Processed: {String.Format("{0:n0}", counter)}"); } } Console.Write($"\r Rows Processed: {String.Format("{0:n0}", counter)}{Environment.NewLine}"); } WriteOutput(cfg, "User", UserStats); WriteOutput(cfg, "Account", AccountStats); }
private static void WriteOutput(Configurator cfg, string UserOrAccount, Dictionary <string, SummaryStatsModel> d) { StringBuilder sb = new StringBuilder(); sb.Append(UserOrAccount); sb.Append(cfg.Delimiter); // hack to bring forward the account to which user is associated if (UserOrAccount == "User") { sb.Append("Account"); sb.Append(cfg.Delimiter); } sb.Append("JobCount"); sb.Append(cfg.Delimiter); sb.Append("AvgCoreHoursPerJob"); sb.Append(cfg.Delimiter); sb.Append("CoreHours"); sb.Append(cfg.Delimiter); sb.Append("GBHoursAllocated"); sb.Append(cfg.Delimiter); sb.Append("GBHoursUsed"); sb.Append(cfg.Delimiter); sb.Append("PercentGBHoursUsed"); sb.Append(cfg.Delimiter); sb.Append("JobIDFirst"); sb.Append(cfg.Delimiter); sb.Append("JobIDLast"); sb.Append(Environment.NewLine); foreach (var item in d) { sb.Append(item.Key); sb.Append(cfg.Delimiter); // hack to bring forward the account to which user is associated if (UserOrAccount == "User") { sb.Append(item.Value.Account); sb.Append(cfg.Delimiter); } sb.Append(item.Value.JobCount); sb.Append(cfg.Delimiter); double AvgCoreHrsPerJob = item.Value.CoreHoursTotal / item.Value.JobCount; sb.Append(Math.Round(AvgCoreHrsPerJob, 1)); sb.Append(cfg.Delimiter); sb.Append(item.Value.CoreHoursTotal); sb.Append(cfg.Delimiter); sb.Append(item.Value.GBHoursReqTotal); sb.Append(cfg.Delimiter); sb.Append(item.Value.GBHoursUsedTotal); sb.Append(cfg.Delimiter); double pct = 0.0; if (item.Value.GBHoursReqTotal > 0) { pct = (item.Value.GBHoursUsedTotal / (double)item.Value.GBHoursReqTotal) * 100.0; } sb.Append(Math.Round(pct, 1)); sb.Append(cfg.Delimiter); sb.Append(item.Value.JobIDFirst); sb.Append(cfg.Delimiter); sb.Append(item.Value.JobIDLast); sb.Append(Environment.NewLine); } string outFile = Path.Combine(cfg.OutputLocation, $"summaryStats-{UserOrAccount}.txt"); File.WriteAllText(outFile, sb.ToString()); Console.WriteLine($"File written: {outFile}"); sb.Clear(); }
private static bool FilterOutThisRecord(string line, Dictionary <string, int> colRef, Configurator cfg) { // All logic that would cause one to ignore a particular line from the job log data var thisRec = line.Split(cfg.Delimiter); // filter unwanted job states if (thisRec[colRef["State"]] == "RUNNING" || thisRec[colRef["State"]] == "SUSPENDED") { return(true); } return(false); }
private static void ProcessWorkSet(List <string> workset, Dictionary <string, int> colRef, Configurator cfg) { ConcurrentBag <string> cb = new ConcurrentBag <string>(); Parallel.ForEach((workset), (line) => { StringBuilder sb = new StringBuilder(); string[] thisRec = line.Split(cfg.Delimiter); // straight copy for the fieldsOfInterest foreach (var item in cfg.ColsOfInterest) { sb.Append($"{thisRec[colRef[item]]}"); sb.Append($"{cfg.Delimiter}"); } // CPUEffeciency: Compare "TotalCPU" with computed field core-walltime; core-walltime = (NCPUS * "Elapsed") int ncpus = int.Parse(thisRec[colRef["NCPUS"]]); double hrsElapsed = Util.ConvertTimeStringToHours(thisRec[colRef["Elapsed"]]); double hrsCoresElapsed = hrsElapsed * ncpus; string corewalltime = Util.ConvertHoursToTimeString(hrsCoresElapsed); double CpuEff = 0; // ComputeCPUEffeciency(thisRec[colRef["TotalCPU"]], corewalltime); sb.Append($"{Math.Round(CpuEff, 2)}"); sb.Append($"{cfg.Delimiter}"); //// MemEffeciency: (MaxRSS / ReqMem) * 100; ReqMem field inlcudes unit, must be stripped and possibly converted string str_ReqMem = thisRec[colRef["ReqMem"]]; int ReqMem = int.Parse(str_ReqMem.Substring(0, (str_ReqMem.Length - 2))); string ReqMemUnits = str_ReqMem.Substring(str_ReqMem.Length - 2); if (ReqMemUnits == "Mc") { ReqMem *= int.Parse(thisRec[colRef["NCPUS"]]); } float MaxRSS = float.Parse(thisRec[colRef["MaxRSS"]]); double MemEff = (double)Math.Round(((MaxRSS / ReqMem) * 100), 2); sb.Append($"{MemEff}"); sb.Append($"{cfg.Delimiter}"); // compute GBHours Requested = (ReqMem / 1000) * Elapsed-Hrs double GBHoursReq = (ReqMem / 1000.0) * hrsElapsed; sb.Append($"{(int)(GBHoursReq)}"); sb.Append($"{cfg.Delimiter}"); JobSize js = Util.JobSizeBin(ncpus, ReqMem, (int)hrsElapsed); var hrsPending = GetPendTime(thisRec[colRef["Submit"]], thisRec[colRef["Start"]]); cb.Add(sb.ToString()); sb.Clear(); }); File.AppendAllLines(cfg.OutputFile, cb); }
private static bool FilterOutThisRecord(string line, Dictionary <string, int> colRef, Configurator cfg) { // All logic that would cause one to ignore a particular line from the job log data var thisRec = line.Split(cfg.Delimiter); int ReqMemThreshold = 3096; // filter unwanted job states if (thisRec[colRef["State"]] == "RUNNING" || thisRec[colRef["State"]] == "SUSPENDED" || thisRec[colRef["State"]] == "OUT_OF_MEMORY") { return(true); } // filter out partitions for which we do not have a max ReqMem configured if (!cfg.PartitionMaxReqMem.ContainsKey(thisRec[colRef["Partition"]])) { return(true); } // filter out records where elapsed time is < 5 minutes string elapsed = thisRec[colRef["Elapsed"]]; if (elapsed.IndexOf("-") == -1) { // if no "-" found within string, then elapsed time was less than one day string[] hhmmss = elapsed.Split(":"); int hh = int.Parse(hhmmss[0]); int mm = int.Parse(hhmmss[1]); if (hh == 0) { if (mm < 5) { return(true); } } } // filter out records where requested memory < ReqMemThreshold string str_ReqMem = thisRec[colRef["ReqMem"]]; if (str_ReqMem.Length < 3) // some cancelled jobs show as "0n" for ReqMem { return(true); } try { int ReqMem = int.Parse(str_ReqMem.Substring(0, (str_ReqMem.Length - 2))); string ReqMemUnits = str_ReqMem.Substring(str_ReqMem.Length - 2); if (ReqMemUnits == "Mn") { // ReqMem is expressed as memory per node. #nodes always = 1 on longleaf if (ReqMem < ReqMemThreshold) { return(true); } } else if (ReqMemUnits == "Mc") { // ReqMem is expressed as memory per core; must multiply by number of cores int NCPUS = int.Parse(thisRec[colRef["NCPUS"]]); if ((ReqMem * NCPUS) < ReqMemThreshold) { return(true); } } else { Console.WriteLine($"*** unrecognized units for ReqMem. str_ReqMem = {str_ReqMem}"); return(true); } } catch (Exception) { Console.WriteLine($"*** error parsing ReqMem. str_ReqMem = {str_ReqMem}"); return(true); } return(false); }