/// <summary> /// Compares data in two usgs rdb files, ignoring comments at the beginning /// </summary> /// <param name="tf1"></param> /// <param name="tf2"></param> /// <returns>true if there are differences in the data</returns> public static bool Diff(TextFile tf1, TextFile tf2) { var diff = System.Math.Abs(tf1.Length - tf2.Length); if ( diff > 5) { Logger.WriteLine("difference in length of files: " + diff); return true; //different number of lines in file } var idx1 = tf1.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR"); var idx2 = tf2.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR"); tf1.DeleteLines(0, idx1); tf2.DeleteLines(0, idx2); return TextFile.Compare(tf1, tf2).Length != 0; }
public static TimeSeriesDatabaseDataSet.RatingTableDataTable ParseRatingTableData(TextFile tf, string cbtt, string pcode) { /* <table border="1" summary="Expanded Rating Table"> <tr align="center"> <th width="80px"><b>ft</b></th> <th width="80px"><b>cfs</b></th> </tr> * * Table Edit Date: 18-SEP-2014 02:22 <tr align="right"><td>2440.01</td><td>-704.00</td></tr> <tr align="right"><td>2440.02</td><td>-704.00</td></tr> <tr align="right"><td>2440.03</td><td>-704.00</td></tr> <tr align="right"><td>2440.04</td><td>-704.00</td></tr> */ TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable(); int idx = tf.IndexOfBothRegex("<th.*</th>", "<th.*</th>"); if (idx >= 0) { t.XUnits = Regex.Match(tf[idx], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value; t.YUnits = Regex.Match(tf[idx + 1], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value; } //<br>Table Edit Date: 18-SEP-2014 02:22<br>Today's Date: 23-SEP-2014 06:54 var idxDate = tf.IndexOfRegex("Table Edit Date:.*<br>"); if (idxDate >= 0) { t.EditDate = Regex.Match(tf[idxDate], "Table Edit Date:(?<date>.*)<br>").Groups[1].Value; } Regex re = new Regex(@"<tr.*?>(?<x>[\d\.\-\+]{1,12})</td><td>(?<y>[\d\.\-\+]{1,12})</td>"); for (int i = idx + 2; i < tf.Length; i++) { var m = re.Match(tf[i]); if (m.Success) { double x, y; if (double.TryParse(m.Groups["x"].Value, out x) ) { if (double.TryParse(m.Groups["y"].Value, out y) && System.Math.Abs(998877 - y) > 0.01 ) { t.AddRatingTableRow(x, y); } } } } t.Name = "Hydromet " + cbtt + " " + pcode; idx = tf.IndexOfRegex("<b>Station .+<br />"); if (idx >= 0) t.Name = Regex.Match(tf[idx], "<b>Station (.+)<br />").Groups[1].Value; return t; }
public void IndexOfRegex() { TextFile tf = new TextFile(data); Assert.AreEqual(9, tf.IndexOfRegex("Jackson$")); Assert.AreEqual(4, tf.IndexOfRegex("object_name: Jackson")); Assert.AreEqual(13, tf.IndexOfRegex("^object_name: Jackson$")); Assert.AreEqual(16, tf.IndexOfBothRegex("^object_name: Jackson$", "^slot_name: Inflow$")); }
/// <summary> /// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is /// instant data a flag column is included to the right of each data column /// </summary> /// <param name="cgi"></param> /// <param name="query"></param> /// <param name="t1"></param> /// <param name="t2"></param> /// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param> /// <param name="endOfMonth"></param> /// <returns></returns> static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false) { //if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year) // t2 = DateTime.Now; int colIndex =1; string flag = " "; string payload = "parameter="+query +"&syer="+t1.Year.ToString() +"&smnth="+t1.Month.ToString() +"&sdy="+t1.Day.ToString() +"&eyer="+t2.Year.ToString() +"&emnth="+t2.Month.ToString() +"&edy="+t2.Day.ToString() +"&format=2"; if (back > 0) { payload = "parameter=" + query + "&back="+back + "&format=2"; } string[] data = Web.GetPage(cgi,payload,HydrometInfoUtility.WebCaching); bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0 || cgi.IndexOf("webmpollcsv") >= 0 || cgi.IndexOf("monthly") >= 0 || cgi.IndexOf("pn-bin/instant") >= 0; TextFile tf = new TextFile(data); int idx1 = tf.IndexOf("BEGIN DATA"); int idx2 = tf.IndexOf("END DATA"); int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date if( idx1 <0 || idx2 <0 || idx2 == idx1+1 || idxDates <0) return new DataTable("No_data"); string strTitles = ""; for (int i = idx1+1; i < idxDates; i++) { strTitles += data[i]; // put wrapped titles back in single line. } string[] columnNames = strTitles.Split(','); DataTable tbl = new DataTable("hydromet"); CreateColumnNames(hasFlag, columnNames, tbl); for(int i=idxDates; i<idx2; i++) { JoinWrappedLines(data, i); string line = data[i]; if (line == "") continue; string[] parts = line.Trim().Split(','); if( parts.Length ==0) continue; DataRow row = tbl.NewRow(); DateTime date = DateTime.Now; //if (!DateTime.TryParse(parts[0], out date)) if( !LinuxUtility.TryParseDateTime(parts[0],out date)) { Logger.WriteLine("Error parsing date " + parts[0]); break; } if( endOfMonth) date = date.Date.EndOfMonth(); row[0] =date; for(int j=1; j<parts.Length; j++) { string strValue = parts[j]; if (strValue.IndexOf("NO RECORD") >= 0 || strValue.IndexOf("MISSING") >= 0 || strValue.IndexOf("998877.00") >=0 || strValue == "") { row[colIndex] = System.DBNull.Value; if (hasFlag) { row[colIndex + 1] = ""; colIndex++; } } else { //Console.WriteLine("str = "+strValue); double val; int index = strValue.IndexOf("."); string str = strValue.Substring(0,index+3); if(hasFlag){ if(strValue.Length>str.Length) flag = strValue.Substring(str.Length,1); else flag = " "; } val = -999; if (!double.TryParse(str, out val)) { Logger.WriteLine("Error converting " + str + " to a number "); } try { row[colIndex] = val; if (hasFlag) { row[colIndex + 1] = flag; colIndex++; } } catch (Exception ex) { MessageBox.Show(ex.Message); } } colIndex++; } tbl.Rows.Add(row); colIndex=1; } //DataSet ds = new DataSet(); //ds.Tables.Add(tbl); return tbl; }