private static string ConvertCSVToShiftFormat(string html, string[] cbtt)
{
/*
* DATE , GAGE HT , CFS , SHIFT ,
4/18/2015 , 1.04 , 152.16 , +0.29 ,
4/21/2015 , 1.07 , 151.5 , +0.27 ,
5/15/2015 , 1.56 , 211.26 , +0.11 ,
6/16/2015 , 1.77 , 118.36 , -0.64 ,
7/25/2015 , 1.67 , 151.3 , -0.35 ,
, , , ,
, , , ,
*/
string[] CRLF = { "\r\n" };
string[] lines = html.Split(CRLF, StringSplitOptions.None);
string cleanFile = "cbtt,pcode,date_measured,discharge,stage,shift\r\n";
var tf = new TextFile(html.Split(new char[]{'\n','\r'}, StringSplitOptions.RemoveEmptyEntries));
for (int i = 0; i < cbtt.Length; i++)
{
var idx = tf.IndexOf(cbtt[i]);
if( idx >=0)
{
var idxDate = tf.IndexOf("DATE", idx);
if( idxDate > idx+5)// date should be within 5 lines of cbtt
{
Console.WriteLine("Error: did not find DATE with cbtt ="+cbtt);
continue;
}
// now parse data until it runs out
for (int j = idxDate+1; j < tf.Length; j++)
{
DateTime t;
var tokens = tf[j].Split(',');
if( tokens.Length < 4)
break;
if( !DateTime.TryParseExact(tokens[0].Trim(), "M/d/yyyy", CultureInfo.InvariantCulture,
DateTimeStyles.None, out t) )
break;
var x = cbtt[i] + ",CH," + t.ToShortDateString() + "," + tokens[2].Trim() + "," + tokens[1].Trim() + "," + tokens[3].Trim();
cleanFile += x + "\r\n";
Console.WriteLine(x);
}
}
else
{
Console.WriteLine("Error: did not find "+cbtt[i]);
}
}
return cleanFile;
}
public void Save(string server, string password)
{
int idx = tf.IndexOf(server);
if (idx < 0)
{
tf.Add(server);
tf.Add(Protect(password));
}
else
{
tf.FileData[idx + 1] = Protect(password);
}
tf.SaveAs(tf.FileName);
}
public void Save(string server, string password)
{
int idx = tf.IndexOf(server);
if (idx < 0)
{
tf.Add(server);
//tf.Add( Protect(password));
tf.Add(StringCipher.Encrypt(password, ""));
}
else
{
tf.FileData[idx + 1] = StringCipher.Encrypt(password, "");
}
tf.SaveAs(tf.FileName);
}
public static string[] Read(string url)
{
TextFile tf = CacheIndex;
int idx = tf.IndexOf(url);
if (idx < 0)
{
throw new FileNotFoundException("error in cache");
}
TextFile df = new TextFile(tf[idx - 1]);
Logger.WriteLine("Cache hit for " + url);
Logger.WriteLine("\"" + tf[idx - 1] + "\"");
return(df.FileData);
}
/***************************************
* returns list of dates from beginning of riverware file in *.rdf format.
**************************************************/
private List <DateTime> ReadDates(Reclamation.Core.TextFile tf, int startingIndex)
{
List <DateTime> rval = new List <DateTime>();
int idx = tf.IndexOfAny(new string[] { "time_steps:", "timesteps" }, startingIndex);
if (idx < 0)
{
throw new Exception("Error looking for 'timesteps' or 'time_steps:' in " + tf.FileName);
}
//timesteps:15706
//time_steps:366
string s = tf[idx];
int i2 = s.IndexOf(":");
s = s.Substring(i2 + 1); // just the number part
int numDates = Convert.ToInt32(s);
int idxToDates = tf.IndexOf("END_RUN_PREAMBLE", idx);
if (idxToDates < 0)
{
throw new Exception("Error looking for END_RUN_PREAMBLE in " + tf.FileName);
}
idxToDates++;
for (int i = 0; i < numDates; i++)
{
DateTime d;
string dateString = tf[idxToDates + i];
dateString = dateString.Replace(" 24:00", " 23:59:59");
if (DateTime.TryParse(dateString, out d))
{
rval.Add(d);
}
else
{
throw new Exception("Can't parse " + tf[idxToDates + i] + " as a date and time");
}
}
return(rval);
}
private static int LookupNumberOfRuns(Reclamation.Core.TextFile tf)
{
int number_of_runs = 1;
//:53
//number_of_runs:53
int index = tf.IndexOf("number_of_runs:");
if (index == -1)
{
Logger.WriteLine("Error: number_of_runs is not defined in file " + tf.FileName);
}
else
{
number_of_runs = Convert.ToInt32(tf[index].Substring(15));
}
return(number_of_runs);
}
public void MultiYearTest()
{
Performance p = new Performance();
string path = Reclamation.Core.Globals.TestDataPath;
for (int i = 1; i <= 2; i++)
{
string dir = @"C:\Temp\dmitest";
Directory.CreateDirectory(dir);
List<string> args = new List<string>();
args.Add(path + "\\RiverWare\\boiseControl.txt");
args.Add(dir);
args.Add("1927-10-04");// date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");
args.Add("12:00");
args.Add("-UXlsFileName=" + path + "\\RiverWare\\BoiseModelData.xls");
args.Add("-sTrace="+i);
if( Directory.Exists(dir))
Directory.Delete(dir,true);
Directory.CreateDirectory(dir);
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = dir + "\\Inflow.Anderson Ranch.txt";
TextFile tf = new TextFile(fn);
int idx = tf.IndexOf("start_date");
if (i == 1) // 1918
{
double val = Convert.ToDouble(tf[idx + 1]);
Assert.AreEqual(429, val, .5);
}
if (i == 2) // 1919
{
double val = Convert.ToDouble(tf[idx + 1]);
Assert.AreEqual(348, val, .5);
}
Directory.Delete(dir, true);
}
}
/// <summary>
/// Extracts data from AquatCalc text file.
/// Example: ParseLine("GAGE ID#")
///
/// looks for line that contains the label "GAGE ID#"
/// then returns the value to the right of the label "GAGE ID#"
/// </summary>
/// <param name="label">text prefix to data. search for this label</param>
/// <returns></returns>
public string ParseLine(string label, int startIndex, int endIndex)
{
TextFile txtFile = this;
int rowIndex = txtFile.IndexOf(label, startIndex);
if (rowIndex < 0 || rowIndex > endIndex)
{
return("");
}
string line = txtFile[rowIndex];
int dataIndex = line.IndexOf(label);
Debug.Assert(dataIndex >= 0, "Error parsing file " + txtFile.FileName + " at line " + rowIndex);
dataIndex += label.Length;
string data = line.Substring(dataIndex).Trim();
return(data);
}
private void ReadSnowDepth(DateTime t1, DateTime t2)
{
/*
--------------------------------------------------------------------------------
Change In
Snow Water Snow Snow Water Change In
Site Name Date Time Equivalent Depth Equivalent Snow Depth
------------------------------------------------------------------------------
ATLANTA SUMMIT 01/07 0000 16.1 58.0
01/08 0000 16.1 57.0 0.0 -1.0
01/09 0000 16.0 57.0 -0.1 0.0
01/10 0000 15.9 57.0 -0.1 0.0
01/11 0000 15.9 56.0 0.0 -1.0
01/12 0000 16.0 56.0 0.1 0.0
01/13 0000 16.8 61.0 0.8 5.0
01/14 0000 17.4 64.0 0.6 3.0
*
--------------------------------------------------------------------------------
Change In
Snow Water Snow Snow Water Change In
Site Name Date Time Equivalent Depth Equivalent Snow Depth
------------------------------------------------------------------------------
JACKSON PEAK 01/07 0000 13.5 50.0
01/08 0000 13.5 48.0 0.0 -2.0
01/09 0000 13.4 49.0 -0.1 1.0
01/10 0000 13.4 48.0 0.0 -1.0
01/11 0000 13.4 48.0 0.0 0.0
01/12 0000 13.5 -99.9 0.1 -99.9
01/13 0000 14.5 -99.9 1.0 -99.9
01/14 0000 15.4 62.0 0.9 -99.9
------------------------------------------------------------------------------
------------------------------------------------------------------------------
*/
var data = Web.GetPage(urlDepth + state + "/" + siteName + ".txt", true);
TextFile tf = new TextFile(data);
var idx = tf.IndexOf("Site Name Date Time Equivalent Depth");
if (idx > 0)
{
string pattern = "^.{23}(?<month>\\d{2})/(?<day>\\d{2})\\s{1}0000\\s*(?<se>[0-9\\.+-]+)\\s*(?<sd>[0-9\\.+-]+)";
for (int i = idx+2; i < idx+10; i++)
{
Regex re = new Regex(pattern);
var m = re.Match(tf[i]);
if (m.Success)
{
int month = int.Parse(m.Groups["month"].Value);
int day = int.Parse(m.Groups["day"].Value);
double sd = double.Parse(m.Groups["sd"].Value);
DateTime t = new DateTime(DateTime.Now.Year, month, day);
if (t >= t1 && t <= t2)
{
if (System.Math.Abs(sd - 99.9) < 0.1)
AddMissing(t);
Add(t, sd);
}
}
}
}
}
public void WaterYearTrace()
{
string path = Globals.TestDataPath;
List<string> args = new List<string>();
//-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919
args.Add(path + "\\RiverWare\\snakeControl.txt");
args.Add("c:\\temp");
args.Add("1927-10-04");// start date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");
args.Add("12:00");
args.Add("1DAY");
args.Add("-UXlsFileName=" +path + "\\RiverWare\\SnakeTestData.xls");
args.Add("-STrace=4");
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = @"c:\temp\Local Inflow.MinidokaToMilner_Milner.txt";
TextFile tf = new TextFile(fn);
File.Delete(fn);
int idx = tf.IndexOf("start_date");
Assert.AreEqual(-2906, Convert.ToDouble(tf[idx + 1]), .6);
Assert.AreEqual(-2864, Convert.ToDouble(tf[idx + 2]), .6);
}
public void WaterYear1931()
{
Performance p = new Performance();
string path = Globals.TestDataPath;
//controlfile tempPath yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm 1DAY -UxlsFileName=file.xls [-STrace=n]");
//-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919
List<string> args = new List<string>();
args.Add(path + "\\RiverWare\\snakeControl.txt");
args.Add("c:\\temp");
args.Add("1927-10-04");// start date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");// end date for riverware
args.Add("12:00");
args.Add("1DAY");
args.Add("-UXlsFileName=" + path + "\\Riverware\\SnakeTestData.xls");
args.Add("-UWaterYear=1931"); // actual xls data begins 10/4/1930 (water year 1931)
args.Add("-UFirstWaterYear=1928");
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = @"c:\temp\Inflow.Jackson.txt";
TextFile tf = new TextFile(fn);
File.Delete(fn);
Assert.AreEqual("# import began on line index: 1113", tf[tf.IndexOf("# import")]);
Assert.AreEqual("start_date: 1927-10-04 24:00", tf[tf.IndexOf("start_date")]);
int idx = tf.IndexOf("start_date");
Assert.AreEqual(691.4, Convert.ToDouble(tf[idx+1]),.1);
Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 2]), .1);
Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 3]), .1);
Assert.AreEqual(4211.4, Convert.ToDouble(tf[idx + 4]), .1);
p.Report("completed FullDataDump()");
}
private static bool CompareHydrometData(TextFile a, TextFile b)
{
int idx1 = a.IndexOf("BEGIN DATA");
a.DeleteLines(0,idx1);
int idx2 = a.IndexOf("END DATA");
a.DeleteLines(idx2, a.Length - 1);
idx1 = b.IndexOf("BEGIN DATA");
b.DeleteLines(0, idx1);
idx2 = b.IndexOf("END DATA");
b.DeleteLines(idx2, b.Length - 1);
if (a.Length != b.Length)
return false;
for (int i = 0; i < a.Length; i++)
{
var x = a[i].Split(',');
var y = b[i].Split(',');
if (x.Length != y.Length)
return false;
for (int j = 0; j < x.Length; j++)
{
if (i == 0) // cbtt parameter
{// allow different spacing
// 'jck fb' or 'jck fb'
RegexOptions options = RegexOptions.None;
Regex regex = new Regex("[ ]{2,}", options);
x[j] = regex.Replace(x[j], " ");
y[j] = regex.Replace(y[j], " ");
}
if (x[j].Trim() != y[j].Trim())
return false;
}
}
return true;
}
protected override void ReadCore(DateTime t1, DateTime t2)
{
// http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Achj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=HTML&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=19&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t=
//http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Aid.chj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=Text+Comma-Delimited&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=15&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t=
//record://los/hf//ir-month/drxzzazd/ time:-1m
string url = "http://www.nwd-wc.usace.army.mil/cgi-bin/dataquery.pl?k=id:[email protected]+record:";
url += m_path + "&sd=01&sm=JAN&sy=2007&ed=23&em=FEB&ey=2007&of=Text+Comma-Delimited";
url = ApplyDatesToURL(t1, t2, url);
string[] results = Reclamation.Core.Web.GetPage(url);
TextFile tf = new TextFile(results);
string msg = "<P>The web server is too loaded right now to service your request. Please wait a few minutes and try your request again. ";
int idx = tf.IndexOf(msg);
if (idx >= 0)
throw new ApplicationException(msg);
string pattern = @"(?<date>\d{1,2}[A-Za-z]{3}\d{4}),(?<hour>\d\d:\d\d),(?<value>[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?)";
Regex re = new Regex(pattern, RegexOptions.Compiled);
Console.WriteLine(pattern);
for (int i = 0; i < results.Length; i++)
{
DateTime t = DateTime.MinValue;
double val = Point.MissingValueFlag;
Match m = re.Match(results[i]);
if (m.Success)
{
GroupCollection g = m.Groups;
if (g["hour"].Value == "24:00")
{
if( DateTime.TryParse(g["date"].Value, out t)
&& double.TryParse(g["value"].Value, out val))
{
t = t.AddDays(1);
AddVal(t, val);
}
}
else
if (DateTime.TryParse(g["date"].Value+" "+g["hour"].Value, out t)
&& double.TryParse(g["value"].Value, out val))
{
AddVal(t, val);
}
else
{
SkipMessage("skipping " + results[i]);
}
}
else
{
SkipMessage("skipping " + results[i]);
}
// Console.WriteLine(results[i]);
}
}
private static string LookupUnits(TextFile tf , string tag)
{
/*
# //RATING_INDEP ROUNDING="2223456782" PARAMETER="Gage height (ft)"
# //RATING_DEP ROUNDING="2222233332" PARAMETER="Discharge (cfs)"
*/
int idx = tf.IndexOf(tag);
if (idx >= 0)
{
string expr = ".*PARAMETER=\"(?<parmeter>)\"";
Regex reg = new Regex(expr);
if (!reg.IsMatch(tf[idx]))
return "";
return reg.Match(tf[idx]).Groups["parameter"].Value;
}
return "";
}
public void ReadFile(string fileName)
{
//m_cbtt = cbtt;
//tring fileName = cbtt + "_" + pcode + ".txt";
var tf = new TextFile(fileName.ToLower());
int idx = tf.IndexOf("BEGIN TABLE"); // hydromet raw dumps
if (idx < 0)
idx = 0;
else
idx += 2;
for (int i = idx; i < tf.Length; i++)
{
var tokens = tf[i].Split(',');
if (tokens.Length == 2)
{
double x, y;
if (double.TryParse(tokens[0], out x)
&& double.TryParse(tokens[1], out y))
{
var r = this.FindByx(x);
if( r == null)
AddRatingTableRow(x, y);
else
{
Console.WriteLine("Warning: Rating table has duplicate independent values. Skipping: "+x+"\n"+fileName);
}
}
}
}
//this.AddRatingTableRow(
}
/// <summary>
/// This method generates the full table from the RDB file
/// </summary>
/// <param name="rdbFile"></param>
private void CreateFullRatingTable(TextFile rdbFile)
{
// Build container DataTable
DataTable ratingTable = new DataTable();
ratingTable.Columns.Add(new DataColumn("Stage", typeof(double)));
ratingTable.Columns.Add(new DataColumn("Shift", typeof(double)));
ratingTable.Columns.Add(new DataColumn("Flow", typeof(double)));
// Save RDB file
var tempFile = Path.GetTempFileName();
rdbFile.SaveAs(tempFile);
rdbFile = new TextFile(tempFile);
// Loop through each RDB file row to find the data
int headerRow = rdbFile.IndexOf("INDEP SHIFT DEP STOR") + 2;
for (int i = headerRow; i < rdbFile.Length; i++)
{
var row = rdbFile[i];
var newRow = ratingTable.NewRow();
var rowVals = rdbFile[i].Split('\t');
newRow["Stage"] = Convert.ToDouble(rowVals[0]);
newRow["Shift"] = Convert.ToDouble(rowVals[1]);
newRow["Flow"] = Convert.ToDouble(rowVals[2]);
ratingTable.Rows.Add(newRow);
}
this.fullRatingTable = ratingTable;
}
/// <summary>
/// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is
/// instant data a flag column is included to the right of each data column
/// </summary>
/// <param name="cgi"></param>
/// <param name="query"></param>
/// <param name="t1"></param>
/// <param name="t2"></param>
/// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param>
/// <param name="endOfMonth"></param>
/// <returns></returns>
static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false)
{
//if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year)
// t2 = DateTime.Now;
int colIndex =1;
string flag = " ";
string payload = "parameter="+query
+"&syer="+t1.Year.ToString()
+"&smnth="+t1.Month.ToString()
+"&sdy="+t1.Day.ToString()
+"&eyer="+t2.Year.ToString()
+"&emnth="+t2.Month.ToString()
+"&edy="+t2.Day.ToString()
+"&format=2";
if (back > 0)
{
payload = "parameter=" + query
+ "&back="+back
+ "&format=2";
}
string[] data = Web.GetPage(cgi,payload,HydrometInfoUtility.WebCaching);
bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0
|| cgi.IndexOf("webmpollcsv") >= 0
|| cgi.IndexOf("monthly") >= 0
|| cgi.IndexOf("pn-bin/instant") >= 0;
TextFile tf = new TextFile(data);
int idx1 = tf.IndexOf("BEGIN DATA");
int idx2 = tf.IndexOf("END DATA");
int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date
if( idx1 <0 || idx2 <0 || idx2 == idx1+1 || idxDates <0)
return new DataTable("No_data");
string strTitles = "";
for (int i = idx1+1; i < idxDates; i++)
{
strTitles += data[i]; // put wrapped titles back in single line.
}
string[] columnNames = strTitles.Split(',');
DataTable tbl = new DataTable("hydromet");
CreateColumnNames(hasFlag, columnNames, tbl);
for(int i=idxDates; i<idx2; i++)
{
JoinWrappedLines(data, i);
string line = data[i];
if (line == "")
continue;
string[] parts = line.Trim().Split(',');
if( parts.Length ==0)
continue;
DataRow row = tbl.NewRow();
DateTime date = DateTime.Now;
//if (!DateTime.TryParse(parts[0], out date))
if( !LinuxUtility.TryParseDateTime(parts[0],out date))
{
Logger.WriteLine("Error parsing date " + parts[0]);
break;
}
if( endOfMonth)
date = date.Date.EndOfMonth();
row[0] =date;
for(int j=1; j<parts.Length; j++)
{
string strValue = parts[j];
if (strValue.IndexOf("NO RECORD") >= 0
|| strValue.IndexOf("MISSING") >= 0
|| strValue.IndexOf("998877.00") >=0
|| strValue == "")
{
row[colIndex] = System.DBNull.Value;
if (hasFlag)
{
row[colIndex + 1] = "";
colIndex++;
}
}
else
{
//Console.WriteLine("str = "+strValue);
double val;
int index = strValue.IndexOf(".");
string str = strValue.Substring(0,index+3);
if(hasFlag){
if(strValue.Length>str.Length)
flag = strValue.Substring(str.Length,1);
else
flag = " ";
}
val = -999;
if (!double.TryParse(str, out val))
{
Logger.WriteLine("Error converting " + str + " to a number ");
}
try
{
row[colIndex] = val;
if (hasFlag)
{
row[colIndex + 1] = flag;
colIndex++;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
colIndex++;
}
tbl.Rows.Add(row);
colIndex=1;
}
//DataSet ds = new DataSet();
//ds.Tables.Add(tbl);
return tbl;
}
