/// <summary>
/// Compares data in two usgs rdb files, ignoring comments at the beginning
/// </summary>
/// <param name="tf1"></param>
/// <param name="tf2"></param>
/// <returns>true if there are differences in the data</returns>
public static bool Diff(TextFile tf1, TextFile tf2)
{
var diff = System.Math.Abs(tf1.Length - tf2.Length);
if ( diff > 5)
{
Logger.WriteLine("difference in length of files: " + diff);
return true; //different number of lines in file
}
var idx1 = tf1.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
var idx2 = tf2.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
tf1.DeleteLines(0, idx1);
tf2.DeleteLines(0, idx2);
return TextFile.Compare(tf1, tf2).Length != 0;
}
public static TimeSeriesDatabaseDataSet.RatingTableDataTable ParseRatingTableData(TextFile tf,
string cbtt, string pcode)
{
/*
<table border="1" summary="Expanded Rating Table"> <tr align="center">
<th width="80px"><b>ft</b></th>
<th width="80px"><b>cfs</b></th>
</tr>
*
* Table Edit Date: 18-SEP-2014 02:22
<tr align="right"><td>2440.01</td><td>-704.00</td></tr>
<tr align="right"><td>2440.02</td><td>-704.00</td></tr>
<tr align="right"><td>2440.03</td><td>-704.00</td></tr>
<tr align="right"><td>2440.04</td><td>-704.00</td></tr>
*/
TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable();
int idx = tf.IndexOfBothRegex("<th.*</th>", "<th.*</th>");
if (idx >= 0)
{
t.XUnits = Regex.Match(tf[idx], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value;
t.YUnits = Regex.Match(tf[idx + 1], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value;
}
//<br>Table Edit Date: 18-SEP-2014 02:22<br>Today's Date: 23-SEP-2014 06:54
var idxDate = tf.IndexOfRegex("Table Edit Date:.*<br>");
if (idxDate >= 0)
{
t.EditDate = Regex.Match(tf[idxDate], "Table Edit Date:(?<date>.*)<br>").Groups[1].Value;
}
Regex re = new Regex(@"<tr.*?>(?<x>[\d\.\-\+]{1,12})</td><td>(?<y>[\d\.\-\+]{1,12})</td>");
for (int i = idx + 2; i < tf.Length; i++)
{
var m = re.Match(tf[i]);
if (m.Success)
{
double x, y;
if (double.TryParse(m.Groups["x"].Value, out x)
)
{
if (double.TryParse(m.Groups["y"].Value, out y)
&& System.Math.Abs(998877 - y) > 0.01
)
{
t.AddRatingTableRow(x, y);
}
}
}
}
t.Name = "Hydromet " + cbtt + " " + pcode;
idx = tf.IndexOfRegex("<b>Station .+<br />");
if (idx >= 0)
t.Name = Regex.Match(tf[idx], "<b>Station (.+)<br />").Groups[1].Value;
return t;
}
public void IndexOfRegex()
{
TextFile tf = new TextFile(data);
Assert.AreEqual(9, tf.IndexOfRegex("Jackson$"));
Assert.AreEqual(4, tf.IndexOfRegex("object_name: Jackson"));
Assert.AreEqual(13, tf.IndexOfRegex("^object_name: Jackson$"));
Assert.AreEqual(16,
tf.IndexOfBothRegex("^object_name: Jackson$", "^slot_name: Inflow$"));
}
/// <summary>
/// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is
/// instant data a flag column is included to the right of each data column
/// </summary>
/// <param name="cgi"></param>
/// <param name="query"></param>
/// <param name="t1"></param>
/// <param name="t2"></param>
/// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param>
/// <param name="endOfMonth"></param>
/// <returns></returns>
static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false)
{
//if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year)
// t2 = DateTime.Now;
int colIndex =1;
string flag = " ";
string payload = "parameter="+query
+"&syer="+t1.Year.ToString()
+"&smnth="+t1.Month.ToString()
+"&sdy="+t1.Day.ToString()
+"&eyer="+t2.Year.ToString()
+"&emnth="+t2.Month.ToString()
+"&edy="+t2.Day.ToString()
+"&format=2";
if (back > 0)
{
payload = "parameter=" + query
+ "&back="+back
+ "&format=2";
}
string[] data = Web.GetPage(cgi,payload,HydrometInfoUtility.WebCaching);
bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0
|| cgi.IndexOf("webmpollcsv") >= 0
|| cgi.IndexOf("monthly") >= 0
|| cgi.IndexOf("pn-bin/instant") >= 0;
TextFile tf = new TextFile(data);
int idx1 = tf.IndexOf("BEGIN DATA");
int idx2 = tf.IndexOf("END DATA");
int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date
if( idx1 <0 || idx2 <0 || idx2 == idx1+1 || idxDates <0)
return new DataTable("No_data");
string strTitles = "";
for (int i = idx1+1; i < idxDates; i++)
{
strTitles += data[i]; // put wrapped titles back in single line.
}
string[] columnNames = strTitles.Split(',');
DataTable tbl = new DataTable("hydromet");
CreateColumnNames(hasFlag, columnNames, tbl);
for(int i=idxDates; i<idx2; i++)
{
JoinWrappedLines(data, i);
string line = data[i];
if (line == "")
continue;
string[] parts = line.Trim().Split(',');
if( parts.Length ==0)
continue;
DataRow row = tbl.NewRow();
DateTime date = DateTime.Now;
//if (!DateTime.TryParse(parts[0], out date))
if( !LinuxUtility.TryParseDateTime(parts[0],out date))
{
Logger.WriteLine("Error parsing date " + parts[0]);
break;
}
if( endOfMonth)
date = date.Date.EndOfMonth();
row[0] =date;
for(int j=1; j<parts.Length; j++)
{
string strValue = parts[j];
if (strValue.IndexOf("NO RECORD") >= 0
|| strValue.IndexOf("MISSING") >= 0
|| strValue.IndexOf("998877.00") >=0
|| strValue == "")
{
row[colIndex] = System.DBNull.Value;
if (hasFlag)
{
row[colIndex + 1] = "";
colIndex++;
}
}
else
{
//Console.WriteLine("str = "+strValue);
double val;
int index = strValue.IndexOf(".");
string str = strValue.Substring(0,index+3);
if(hasFlag){
if(strValue.Length>str.Length)
flag = strValue.Substring(str.Length,1);
else
flag = " ";
}
val = -999;
if (!double.TryParse(str, out val))
{
Logger.WriteLine("Error converting " + str + " to a number ");
}
try
{
row[colIndex] = val;
if (hasFlag)
{
row[colIndex + 1] = flag;
colIndex++;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
colIndex++;
}
tbl.Rows.Add(row);
colIndex=1;
}
//DataSet ds = new DataSet();
//ds.Tables.Add(tbl);
return tbl;
}
