private static string ConvertCSVToShiftFormat(string html, string[] cbtt) { /* * DATE , GAGE HT , CFS , SHIFT , 4/18/2015 , 1.04 , 152.16 , +0.29 , 4/21/2015 , 1.07 , 151.5 , +0.27 , 5/15/2015 , 1.56 , 211.26 , +0.11 , 6/16/2015 , 1.77 , 118.36 , -0.64 , 7/25/2015 , 1.67 , 151.3 , -0.35 , , , , , , , , , */ string[] CRLF = { "\r\n" }; string[] lines = html.Split(CRLF, StringSplitOptions.None); string cleanFile = "cbtt,pcode,date_measured,discharge,stage,shift\r\n"; var tf = new TextFile(html.Split(new char[]{'\n','\r'}, StringSplitOptions.RemoveEmptyEntries)); for (int i = 0; i < cbtt.Length; i++) { var idx = tf.IndexOf(cbtt[i]); if( idx >=0) { var idxDate = tf.IndexOf("DATE", idx); if( idxDate > idx+5)// date should be within 5 lines of cbtt { Console.WriteLine("Error: did not find DATE with cbtt ="+cbtt); continue; } // now parse data until it runs out for (int j = idxDate+1; j < tf.Length; j++) { DateTime t; var tokens = tf[j].Split(','); if( tokens.Length < 4) break; if( !DateTime.TryParseExact(tokens[0].Trim(), "M/d/yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out t) ) break; var x = cbtt[i] + ",CH," + t.ToShortDateString() + "," + tokens[2].Trim() + "," + tokens[1].Trim() + "," + tokens[3].Trim(); cleanFile += x + "\r\n"; Console.WriteLine(x); } } else { Console.WriteLine("Error: did not find "+cbtt[i]); } } return cleanFile; }
public void Save(string server, string password) { int idx = tf.IndexOf(server); if (idx < 0) { tf.Add(server); tf.Add(Protect(password)); } else { tf.FileData[idx + 1] = Protect(password); } tf.SaveAs(tf.FileName); }
public void Save(string server, string password) { int idx = tf.IndexOf(server); if (idx < 0) { tf.Add(server); //tf.Add( Protect(password)); tf.Add(StringCipher.Encrypt(password, "")); } else { tf.FileData[idx + 1] = StringCipher.Encrypt(password, ""); } tf.SaveAs(tf.FileName); }
public static string[] Read(string url) { TextFile tf = CacheIndex; int idx = tf.IndexOf(url); if (idx < 0) { throw new FileNotFoundException("error in cache"); } TextFile df = new TextFile(tf[idx - 1]); Logger.WriteLine("Cache hit for " + url); Logger.WriteLine("\"" + tf[idx - 1] + "\""); return(df.FileData); }
/*************************************** * returns list of dates from beginning of riverware file in *.rdf format. **************************************************/ private List <DateTime> ReadDates(Reclamation.Core.TextFile tf, int startingIndex) { List <DateTime> rval = new List <DateTime>(); int idx = tf.IndexOfAny(new string[] { "time_steps:", "timesteps" }, startingIndex); if (idx < 0) { throw new Exception("Error looking for 'timesteps' or 'time_steps:' in " + tf.FileName); } //timesteps:15706 //time_steps:366 string s = tf[idx]; int i2 = s.IndexOf(":"); s = s.Substring(i2 + 1); // just the number part int numDates = Convert.ToInt32(s); int idxToDates = tf.IndexOf("END_RUN_PREAMBLE", idx); if (idxToDates < 0) { throw new Exception("Error looking for END_RUN_PREAMBLE in " + tf.FileName); } idxToDates++; for (int i = 0; i < numDates; i++) { DateTime d; string dateString = tf[idxToDates + i]; dateString = dateString.Replace(" 24:00", " 23:59:59"); if (DateTime.TryParse(dateString, out d)) { rval.Add(d); } else { throw new Exception("Can't parse " + tf[idxToDates + i] + " as a date and time"); } } return(rval); }
private static int LookupNumberOfRuns(Reclamation.Core.TextFile tf) { int number_of_runs = 1; //:53 //number_of_runs:53 int index = tf.IndexOf("number_of_runs:"); if (index == -1) { Logger.WriteLine("Error: number_of_runs is not defined in file " + tf.FileName); } else { number_of_runs = Convert.ToInt32(tf[index].Substring(15)); } return(number_of_runs); }
public void MultiYearTest() { Performance p = new Performance(); string path = Reclamation.Core.Globals.TestDataPath; for (int i = 1; i <= 2; i++) { string dir = @"C:\Temp\dmitest"; Directory.CreateDirectory(dir); List<string> args = new List<string>(); args.Add(path + "\\RiverWare\\boiseControl.txt"); args.Add(dir); args.Add("1927-10-04");// date for RiverWare args.Add("12:00"); args.Add("1928-07-01"); args.Add("12:00"); args.Add("-UXlsFileName=" + path + "\\RiverWare\\BoiseModelData.xls"); args.Add("-sTrace="+i); if( Directory.Exists(dir)) Directory.Delete(dir,true); Directory.CreateDirectory(dir); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = dir + "\\Inflow.Anderson Ranch.txt"; TextFile tf = new TextFile(fn); int idx = tf.IndexOf("start_date"); if (i == 1) // 1918 { double val = Convert.ToDouble(tf[idx + 1]); Assert.AreEqual(429, val, .5); } if (i == 2) // 1919 { double val = Convert.ToDouble(tf[idx + 1]); Assert.AreEqual(348, val, .5); } Directory.Delete(dir, true); } }
/// <summary> /// Extracts data from AquatCalc text file. /// Example: ParseLine("GAGE ID#") /// /// looks for line that contains the label "GAGE ID#" /// then returns the value to the right of the label "GAGE ID#" /// </summary> /// <param name="label">text prefix to data. search for this label</param> /// <returns></returns> public string ParseLine(string label, int startIndex, int endIndex) { TextFile txtFile = this; int rowIndex = txtFile.IndexOf(label, startIndex); if (rowIndex < 0 || rowIndex > endIndex) { return(""); } string line = txtFile[rowIndex]; int dataIndex = line.IndexOf(label); Debug.Assert(dataIndex >= 0, "Error parsing file " + txtFile.FileName + " at line " + rowIndex); dataIndex += label.Length; string data = line.Substring(dataIndex).Trim(); return(data); }
private void ReadSnowDepth(DateTime t1, DateTime t2) { /* -------------------------------------------------------------------------------- Change In Snow Water Snow Snow Water Change In Site Name Date Time Equivalent Depth Equivalent Snow Depth ------------------------------------------------------------------------------ ATLANTA SUMMIT 01/07 0000 16.1 58.0 01/08 0000 16.1 57.0 0.0 -1.0 01/09 0000 16.0 57.0 -0.1 0.0 01/10 0000 15.9 57.0 -0.1 0.0 01/11 0000 15.9 56.0 0.0 -1.0 01/12 0000 16.0 56.0 0.1 0.0 01/13 0000 16.8 61.0 0.8 5.0 01/14 0000 17.4 64.0 0.6 3.0 * -------------------------------------------------------------------------------- Change In Snow Water Snow Snow Water Change In Site Name Date Time Equivalent Depth Equivalent Snow Depth ------------------------------------------------------------------------------ JACKSON PEAK 01/07 0000 13.5 50.0 01/08 0000 13.5 48.0 0.0 -2.0 01/09 0000 13.4 49.0 -0.1 1.0 01/10 0000 13.4 48.0 0.0 -1.0 01/11 0000 13.4 48.0 0.0 0.0 01/12 0000 13.5 -99.9 0.1 -99.9 01/13 0000 14.5 -99.9 1.0 -99.9 01/14 0000 15.4 62.0 0.9 -99.9 ------------------------------------------------------------------------------ ------------------------------------------------------------------------------ */ var data = Web.GetPage(urlDepth + state + "/" + siteName + ".txt", true); TextFile tf = new TextFile(data); var idx = tf.IndexOf("Site Name Date Time Equivalent Depth"); if (idx > 0) { string pattern = "^.{23}(?<month>\\d{2})/(?<day>\\d{2})\\s{1}0000\\s*(?<se>[0-9\\.+-]+)\\s*(?<sd>[0-9\\.+-]+)"; for (int i = idx+2; i < idx+10; i++) { Regex re = new Regex(pattern); var m = re.Match(tf[i]); if (m.Success) { int month = int.Parse(m.Groups["month"].Value); int day = int.Parse(m.Groups["day"].Value); double sd = double.Parse(m.Groups["sd"].Value); DateTime t = new DateTime(DateTime.Now.Year, month, day); if (t >= t1 && t <= t2) { if (System.Math.Abs(sd - 99.9) < 0.1) AddMissing(t); Add(t, sd); } } } } }
public void WaterYearTrace() { string path = Globals.TestDataPath; List<string> args = new List<string>(); //-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919 args.Add(path + "\\RiverWare\\snakeControl.txt"); args.Add("c:\\temp"); args.Add("1927-10-04");// start date for RiverWare args.Add("12:00"); args.Add("1928-07-01"); args.Add("12:00"); args.Add("1DAY"); args.Add("-UXlsFileName=" +path + "\\RiverWare\\SnakeTestData.xls"); args.Add("-STrace=4"); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = @"c:\temp\Local Inflow.MinidokaToMilner_Milner.txt"; TextFile tf = new TextFile(fn); File.Delete(fn); int idx = tf.IndexOf("start_date"); Assert.AreEqual(-2906, Convert.ToDouble(tf[idx + 1]), .6); Assert.AreEqual(-2864, Convert.ToDouble(tf[idx + 2]), .6); }
public void WaterYear1931() { Performance p = new Performance(); string path = Globals.TestDataPath; //controlfile tempPath yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm 1DAY -UxlsFileName=file.xls [-STrace=n]"); //-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919 List<string> args = new List<string>(); args.Add(path + "\\RiverWare\\snakeControl.txt"); args.Add("c:\\temp"); args.Add("1927-10-04");// start date for RiverWare args.Add("12:00"); args.Add("1928-07-01");// end date for riverware args.Add("12:00"); args.Add("1DAY"); args.Add("-UXlsFileName=" + path + "\\Riverware\\SnakeTestData.xls"); args.Add("-UWaterYear=1931"); // actual xls data begins 10/4/1930 (water year 1931) args.Add("-UFirstWaterYear=1928"); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = @"c:\temp\Inflow.Jackson.txt"; TextFile tf = new TextFile(fn); File.Delete(fn); Assert.AreEqual("# import began on line index: 1113", tf[tf.IndexOf("# import")]); Assert.AreEqual("start_date: 1927-10-04 24:00", tf[tf.IndexOf("start_date")]); int idx = tf.IndexOf("start_date"); Assert.AreEqual(691.4, Convert.ToDouble(tf[idx+1]),.1); Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 2]), .1); Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 3]), .1); Assert.AreEqual(4211.4, Convert.ToDouble(tf[idx + 4]), .1); p.Report("completed FullDataDump()"); }
private static bool CompareHydrometData(TextFile a, TextFile b) { int idx1 = a.IndexOf("BEGIN DATA"); a.DeleteLines(0,idx1); int idx2 = a.IndexOf("END DATA"); a.DeleteLines(idx2, a.Length - 1); idx1 = b.IndexOf("BEGIN DATA"); b.DeleteLines(0, idx1); idx2 = b.IndexOf("END DATA"); b.DeleteLines(idx2, b.Length - 1); if (a.Length != b.Length) return false; for (int i = 0; i < a.Length; i++) { var x = a[i].Split(','); var y = b[i].Split(','); if (x.Length != y.Length) return false; for (int j = 0; j < x.Length; j++) { if (i == 0) // cbtt parameter {// allow different spacing // 'jck fb' or 'jck fb' RegexOptions options = RegexOptions.None; Regex regex = new Regex("[ ]{2,}", options); x[j] = regex.Replace(x[j], " "); y[j] = regex.Replace(y[j], " "); } if (x[j].Trim() != y[j].Trim()) return false; } } return true; }
protected override void ReadCore(DateTime t1, DateTime t2) { // http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Achj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=HTML&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=19&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t= //http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Aid.chj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=Text+Comma-Delimited&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=15&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t= //record://los/hf//ir-month/drxzzazd/ time:-1m string url = "http://www.nwd-wc.usace.army.mil/cgi-bin/dataquery.pl?k=id:[email protected]+record:"; url += m_path + "&sd=01&sm=JAN&sy=2007&ed=23&em=FEB&ey=2007&of=Text+Comma-Delimited"; url = ApplyDatesToURL(t1, t2, url); string[] results = Reclamation.Core.Web.GetPage(url); TextFile tf = new TextFile(results); string msg = "<P>The web server is too loaded right now to service your request. Please wait a few minutes and try your request again. "; int idx = tf.IndexOf(msg); if (idx >= 0) throw new ApplicationException(msg); string pattern = @"(?<date>\d{1,2}[A-Za-z]{3}\d{4}),(?<hour>\d\d:\d\d),(?<value>[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?)"; Regex re = new Regex(pattern, RegexOptions.Compiled); Console.WriteLine(pattern); for (int i = 0; i < results.Length; i++) { DateTime t = DateTime.MinValue; double val = Point.MissingValueFlag; Match m = re.Match(results[i]); if (m.Success) { GroupCollection g = m.Groups; if (g["hour"].Value == "24:00") { if( DateTime.TryParse(g["date"].Value, out t) && double.TryParse(g["value"].Value, out val)) { t = t.AddDays(1); AddVal(t, val); } } else if (DateTime.TryParse(g["date"].Value+" "+g["hour"].Value, out t) && double.TryParse(g["value"].Value, out val)) { AddVal(t, val); } else { SkipMessage("skipping " + results[i]); } } else { SkipMessage("skipping " + results[i]); } // Console.WriteLine(results[i]); } }
private static string LookupUnits(TextFile tf , string tag) { /* # //RATING_INDEP ROUNDING="2223456782" PARAMETER="Gage height (ft)" # //RATING_DEP ROUNDING="2222233332" PARAMETER="Discharge (cfs)" */ int idx = tf.IndexOf(tag); if (idx >= 0) { string expr = ".*PARAMETER=\"(?<parmeter>)\""; Regex reg = new Regex(expr); if (!reg.IsMatch(tf[idx])) return ""; return reg.Match(tf[idx]).Groups["parameter"].Value; } return ""; }
public void ReadFile(string fileName) { //m_cbtt = cbtt; //tring fileName = cbtt + "_" + pcode + ".txt"; var tf = new TextFile(fileName.ToLower()); int idx = tf.IndexOf("BEGIN TABLE"); // hydromet raw dumps if (idx < 0) idx = 0; else idx += 2; for (int i = idx; i < tf.Length; i++) { var tokens = tf[i].Split(','); if (tokens.Length == 2) { double x, y; if (double.TryParse(tokens[0], out x) && double.TryParse(tokens[1], out y)) { var r = this.FindByx(x); if( r == null) AddRatingTableRow(x, y); else { Console.WriteLine("Warning: Rating table has duplicate independent values. Skipping: "+x+"\n"+fileName); } } } } //this.AddRatingTableRow( }
/// <summary> /// This method generates the full table from the RDB file /// </summary> /// <param name="rdbFile"></param> private void CreateFullRatingTable(TextFile rdbFile) { // Build container DataTable DataTable ratingTable = new DataTable(); ratingTable.Columns.Add(new DataColumn("Stage", typeof(double))); ratingTable.Columns.Add(new DataColumn("Shift", typeof(double))); ratingTable.Columns.Add(new DataColumn("Flow", typeof(double))); // Save RDB file var tempFile = Path.GetTempFileName(); rdbFile.SaveAs(tempFile); rdbFile = new TextFile(tempFile); // Loop through each RDB file row to find the data int headerRow = rdbFile.IndexOf("INDEP SHIFT DEP STOR") + 2; for (int i = headerRow; i < rdbFile.Length; i++) { var row = rdbFile[i]; var newRow = ratingTable.NewRow(); var rowVals = rdbFile[i].Split('\t'); newRow["Stage"] = Convert.ToDouble(rowVals[0]); newRow["Shift"] = Convert.ToDouble(rowVals[1]); newRow["Flow"] = Convert.ToDouble(rowVals[2]); ratingTable.Rows.Add(newRow); } this.fullRatingTable = ratingTable; }
/// <summary> /// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is /// instant data a flag column is included to the right of each data column /// </summary> /// <param name="cgi"></param> /// <param name="query"></param> /// <param name="t1"></param> /// <param name="t2"></param> /// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param> /// <param name="endOfMonth"></param> /// <returns></returns> static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false) { //if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year) // t2 = DateTime.Now; int colIndex =1; string flag = " "; string payload = "parameter="+query +"&syer="+t1.Year.ToString() +"&smnth="+t1.Month.ToString() +"&sdy="+t1.Day.ToString() +"&eyer="+t2.Year.ToString() +"&emnth="+t2.Month.ToString() +"&edy="+t2.Day.ToString() +"&format=2"; if (back > 0) { payload = "parameter=" + query + "&back="+back + "&format=2"; } string[] data = Web.GetPage(cgi,payload,HydrometInfoUtility.WebCaching); bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0 || cgi.IndexOf("webmpollcsv") >= 0 || cgi.IndexOf("monthly") >= 0 || cgi.IndexOf("pn-bin/instant") >= 0; TextFile tf = new TextFile(data); int idx1 = tf.IndexOf("BEGIN DATA"); int idx2 = tf.IndexOf("END DATA"); int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date if( idx1 <0 || idx2 <0 || idx2 == idx1+1 || idxDates <0) return new DataTable("No_data"); string strTitles = ""; for (int i = idx1+1; i < idxDates; i++) { strTitles += data[i]; // put wrapped titles back in single line. } string[] columnNames = strTitles.Split(','); DataTable tbl = new DataTable("hydromet"); CreateColumnNames(hasFlag, columnNames, tbl); for(int i=idxDates; i<idx2; i++) { JoinWrappedLines(data, i); string line = data[i]; if (line == "") continue; string[] parts = line.Trim().Split(','); if( parts.Length ==0) continue; DataRow row = tbl.NewRow(); DateTime date = DateTime.Now; //if (!DateTime.TryParse(parts[0], out date)) if( !LinuxUtility.TryParseDateTime(parts[0],out date)) { Logger.WriteLine("Error parsing date " + parts[0]); break; } if( endOfMonth) date = date.Date.EndOfMonth(); row[0] =date; for(int j=1; j<parts.Length; j++) { string strValue = parts[j]; if (strValue.IndexOf("NO RECORD") >= 0 || strValue.IndexOf("MISSING") >= 0 || strValue.IndexOf("998877.00") >=0 || strValue == "") { row[colIndex] = System.DBNull.Value; if (hasFlag) { row[colIndex + 1] = ""; colIndex++; } } else { //Console.WriteLine("str = "+strValue); double val; int index = strValue.IndexOf("."); string str = strValue.Substring(0,index+3); if(hasFlag){ if(strValue.Length>str.Length) flag = strValue.Substring(str.Length,1); else flag = " "; } val = -999; if (!double.TryParse(str, out val)) { Logger.WriteLine("Error converting " + str + " to a number "); } try { row[colIndex] = val; if (hasFlag) { row[colIndex + 1] = flag; colIndex++; } } catch (Exception ex) { MessageBox.Show(ex.Message); } } colIndex++; } tbl.Rows.Add(row); colIndex=1; } //DataSet ds = new DataSet(); //ds.Tables.Add(tbl); return tbl; }