public void WaterYear1931()
{
Performance p = new Performance();
string path = Globals.TestDataPath;
//controlfile tempPath yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm 1DAY -UxlsFileName=file.xls [-STrace=n]");
//-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919
List<string> args = new List<string>();
args.Add(path + "\\RiverWare\\snakeControl.txt");
args.Add("c:\\temp");
args.Add("1927-10-04");// start date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");// end date for riverware
args.Add("12:00");
args.Add("1DAY");
args.Add("-UXlsFileName=" + path + "\\Riverware\\SnakeTestData.xls");
args.Add("-UWaterYear=1931"); // actual xls data begins 10/4/1930 (water year 1931)
args.Add("-UFirstWaterYear=1928");
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = @"c:\temp\Inflow.Jackson.txt";
TextFile tf = new TextFile(fn);
File.Delete(fn);
Assert.AreEqual("# import began on line index: 1113", tf[tf.IndexOf("# import")]);
Assert.AreEqual("start_date: 1927-10-04 24:00", tf[tf.IndexOf("start_date")]);
int idx = tf.IndexOf("start_date");
Assert.AreEqual(691.4, Convert.ToDouble(tf[idx+1]),.1);
Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 2]), .1);
Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 3]), .1);
Assert.AreEqual(4211.4, Convert.ToDouble(tf[idx + 4]), .1);
p.Report("completed FullDataDump()");
}
public static TimeSeriesDatabaseDataSet.RatingTableDataTable GetRatingTable(string siteNumber)
{
//string url = "http://waterdata.usgs.gov/nwisweb/data/exsa_rat/13236500.rdb";
string url = "http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=13081500&file_type=exsa";
url = url.Replace("13081500", siteNumber);
string[] data = Web.GetPage(url);
TextFile tf = new TextFile(data);
UsgsRDBFile rdb = new UsgsRDBFile(data);
TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable();
t.XUnits = LookupUnits(tf,"# //RATING_INDEP");
t.YUnits = LookupUnits(tf,"# //RATING_DEP");
for (int i = 0; i < rdb.Rows.Count; i++)
{
double x = Convert.ToDouble(rdb.Rows[i]["indep"]);
double y = Convert.ToDouble(rdb.Rows[i]["dep"]);
t.AddRatingTableRow(x, y);
}
t.Name = "Usgs " + siteNumber;
return t;
}
public DataQueryInfo(HecDssPath path)
{
string url = "http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=" + path.B;
results = Reclamation.Core.Web.GetPage(url);
tf = new TextFile(results);
this.path = path;
name = GetName();
ParseAttributes();
}
private static string ConvertCSVToShiftFormat(string html, string[] cbtt)
{
/*
* DATE , GAGE HT , CFS , SHIFT ,
4/18/2015 , 1.04 , 152.16 , +0.29 ,
4/21/2015 , 1.07 , 151.5 , +0.27 ,
5/15/2015 , 1.56 , 211.26 , +0.11 ,
6/16/2015 , 1.77 , 118.36 , -0.64 ,
7/25/2015 , 1.67 , 151.3 , -0.35 ,
, , , ,
, , , ,
*/
string[] CRLF = { "\r\n" };
string[] lines = html.Split(CRLF, StringSplitOptions.None);
string cleanFile = "cbtt,pcode,date_measured,discharge,stage,shift\r\n";
var tf = new TextFile(html.Split(new char[]{'\n','\r'}, StringSplitOptions.RemoveEmptyEntries));
for (int i = 0; i < cbtt.Length; i++)
{
var idx = tf.IndexOf(cbtt[i]);
if( idx >=0)
{
var idxDate = tf.IndexOf("DATE", idx);
if( idxDate > idx+5)// date should be within 5 lines of cbtt
{
Console.WriteLine("Error: did not find DATE with cbtt ="+cbtt);
continue;
}
// now parse data until it runs out
for (int j = idxDate+1; j < tf.Length; j++)
{
DateTime t;
var tokens = tf[j].Split(',');
if( tokens.Length < 4)
break;
if( !DateTime.TryParseExact(tokens[0].Trim(), "M/d/yyyy", CultureInfo.InvariantCulture,
DateTimeStyles.None, out t) )
break;
var x = cbtt[i] + ",CH," + t.ToShortDateString() + "," + tokens[2].Trim() + "," + tokens[1].Trim() + "," + tokens[3].Trim();
cleanFile += x + "\r\n";
Console.WriteLine(x);
}
}
else
{
Console.WriteLine("Error: did not find "+cbtt[i]);
}
}
return cleanFile;
}
/// <summary>
/// Reads shef .A file and extracts all shef locations and codes that match
/// simple, only reads single parameter per line
/// </summary>
/// <param name="filename">input file to read</param>
/// <param name="shefloc">shef location to search for</param>
/// <param name="shefCode">shefcode to search for</param>
/// <returns></returns>
public static Series ReadSimpleShefA(string filename, string shefloc, string shefCode)
{
//.A BRWLD 150309 PD DH2315 /HE1H .00
//.A BRWLD 150309 PD DH2315 /HE2H .19
var rval = new Series();
TextFile tf = new TextFile(filename);
for (int i = 0; i < tf.Length; i++)
{
var m = ShefASimpleRegex.Match(tf[i]);
if (m.Success)
{
var loc = m.Groups["shefloc"].Value;
var sc = m.Groups["shefcode"].Value;
var yy = m.Groups["year"].Value;
var mm = m.Groups["month"].Value;
var dd = m.Groups["day"].Value;
var hh = m.Groups["hour"].Value;
var min = m.Groups["min"].Value;
var val = m.Groups["value"].Value;
if (shefCode == sc
&& loc == shefloc)
{
int year, month, day, hour,minute;
double num = Point.MissingValueFlag; ;
if (int.TryParse(yy, out year)
&& int.TryParse(mm, out month)
&& int.TryParse(dd, out day)
&& int.TryParse(hh, out hour)
&& int.TryParse(min, out minute)
&& double.TryParse(val,out num))
{
var t = new DateTime(year + 2000, month, day, hour, minute,0);
if( rval.IndexOf(t)>=0)
{
Console.WriteLine("Skipping duplicate "+t.ToString()+" :" +num);
continue;
}
rval.Add(t, num);
}
}
}
}
return rval;
}
/***************************************
* returns list of dates from beginning of riverware file in *.rdf format.
**************************************************/
private List <DateTime> ReadDates(Reclamation.Core.TextFile tf, int startingIndex)
{
List <DateTime> rval = new List <DateTime>();
int idx = tf.IndexOfAny(new string[] { "time_steps:", "timesteps" }, startingIndex);
if (idx < 0)
{
throw new Exception("Error looking for 'timesteps' or 'time_steps:' in " + tf.FileName);
}
//timesteps:15706
//time_steps:366
string s = tf[idx];
int i2 = s.IndexOf(":");
s = s.Substring(i2 + 1); // just the number part
int numDates = Convert.ToInt32(s);
int idxToDates = tf.IndexOf("END_RUN_PREAMBLE", idx);
if (idxToDates < 0)
{
throw new Exception("Error looking for END_RUN_PREAMBLE in " + tf.FileName);
}
idxToDates++;
for (int i = 0; i < numDates; i++)
{
DateTime d;
string dateString = tf[idxToDates + i];
dateString = dateString.Replace(" 24:00", " 23:59:59");
if (DateTime.TryParse(dateString, out d))
{
rval.Add(d);
}
else
{
throw new Exception("Can't parse " + tf[idxToDates + i] + " as a date and time");
}
}
return(rval);
}
/// <summary>
/// Compares data in two usgs rdb files, ignoring comments at the beginning
/// </summary>
/// <param name="tf1"></param>
/// <param name="tf2"></param>
/// <returns>true if there are differences in the data</returns>
public static bool Diff(TextFile tf1, TextFile tf2)
{
var diff = System.Math.Abs(tf1.Length - tf2.Length);
if ( diff > 5)
{
Logger.WriteLine("difference in length of files: " + diff);
return true; //different number of lines in file
}
var idx1 = tf1.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
var idx2 = tf2.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
tf1.DeleteLines(0, idx1);
tf2.DeleteLines(0, idx2);
return TextFile.Compare(tf1, tf2).Length != 0;
}
private static int LookupNumberOfRuns(Reclamation.Core.TextFile tf)
{
int number_of_runs = 1;
//:53
//number_of_runs:53
int index = tf.IndexOf("number_of_runs:");
if (index == -1)
{
Logger.WriteLine("Error: number_of_runs is not defined in file " + tf.FileName);
}
else
{
number_of_runs = Convert.ToInt32(tf[index].Substring(15));
}
return(number_of_runs);
}
public void CGI_Daily_CleanerDesign()
{
string payload = "parameter=luc fb,luc af&start=2016-5-19&end=2016-5-24&format=csv";
var fn = RunTest(payload, TimeInterval.Daily);
/* DATE , LUC FB , LUC AF
5/24/2016 4:10:00 PM: CreateSQL
5/24/2016 4:10:00 PM: list of 2 series
05/19/2016,, 253349.11
05/20/2016,, 253698.69
05/21/2016,, 254910.41
05/22/2016,, 255826.27
05/23/2016,, 255933.44
*/
TextFile tf = new TextFile(fn);
int idx = tf.IndexBeginningWith("05/19/2016", 0);
var s = tf[idx].Split(',')[2];
double d = Convert.ToDouble(s);
Assert.AreEqual(253349.11, d, 1.0);
}
public void MultiYearTest()
{
Performance p = new Performance();
string path = Reclamation.Core.Globals.TestDataPath;
for (int i = 1; i <= 2; i++)
{
string dir = @"C:\Temp\dmitest";
Directory.CreateDirectory(dir);
List<string> args = new List<string>();
args.Add(path + "\\RiverWare\\boiseControl.txt");
args.Add(dir);
args.Add("1927-10-04");// date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");
args.Add("12:00");
args.Add("-UXlsFileName=" + path + "\\RiverWare\\BoiseModelData.xls");
args.Add("-sTrace="+i);
if( Directory.Exists(dir))
Directory.Delete(dir,true);
Directory.CreateDirectory(dir);
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = dir + "\\Inflow.Anderson Ranch.txt";
TextFile tf = new TextFile(fn);
int idx = tf.IndexOf("start_date");
if (i == 1) // 1918
{
double val = Convert.ToDouble(tf[idx + 1]);
Assert.AreEqual(429, val, .5);
}
if (i == 2) // 1919
{
double val = Convert.ToDouble(tf[idx + 1]);
Assert.AreEqual(348, val, .5);
}
Directory.Delete(dir, true);
}
}
/// <summary>
/// Main constructor for this class
/// </summary>
/// <param name="idNumber"></param>
public UsgsRatingTable(string idNumber)
{
this.idNumber = idNumber;
// Get and assign RDB file from the web
string nwisURL = "http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=XXXXXXXX&file_type=exsa";
downloadURL = nwisURL.Replace("XXXXXXXX", idNumber);
var newData = Web.GetPage(nwisURL.Replace("XXXXXXXX", idNumber));
if (newData.Count() == 0)
{ throw new Exception("NWIS data not found. Check inputs or retry later."); }
webRdbTable = new TextFile(newData);
webRdbTable.DeleteLine(webRdbTable.Length - 1); //last line from web is blank and the exisitng RDB does not have an empty last line
// Get and assign RDB file properties
this.units = webRdbTable.ReadString("LABEL=").Replace("\"", "");
this.stationName = webRdbTable.ReadString("STATION NAME=").Replace("\"", "");
this.timeZone = webRdbTable.ReadString("TIME_ZONE=").Replace("\"", "");
this.ratingTableVersion = webRdbTable.ReadString("RATING ID=").Replace("\"", "");
this.ratingTableComments = webRdbTable.ReadString("COMMENT=").Replace("\"", "");
this.ratingTableExpansion = webRdbTable.ReadString("RATING EXPANSION=").Replace("\"", "");
}
public static void CompareLinuxToVMSCGI(string payload,TimeInterval interval= TimeInterval.Irregular)
{
//Program.Main(new string[] { "--cgi=instant", "--payload=?"+payload });
TimeSeriesDatabase db = TimeSeriesDatabase.InitDatabase(new Arguments(new string[]{}));
WebTimeSeriesWriter c = new WebTimeSeriesWriter(db,interval,payload);
var fn = FileUtility.GetTempFileName(".txt");
Console.WriteLine("linux temp file:"+fn);
c.Run(fn);
TextFile tf = new TextFile(fn);
tf.DeleteLines(0, 1);
var fnhyd0 = FileUtility.GetTempFileName(".txt");
Console.WriteLine("vms temp file:" + fnhyd0);
string url = "https://www.usbr.gov/pn-bin/webarccsv.pl?";
if( interval == TimeInterval.Irregular || interval == TimeInterval.Hourly)
url = "https://www.usbr.gov/pn-bin/webdaycsv.pl?";
Web.GetFile(url + payload, fnhyd0);
var tf2 = new TextFile(fnhyd0);
if (!CompareHydrometData(tf, tf2))
{
// do detailed comparision.
var diff = TextFile.Compare(tf, tf2);
if (diff.Length > 0)
{
for (int i = 0; i < tf.Length; i++)
{
Console.WriteLine(tf[i]);
}
}
Assert.IsTrue(diff.Length == 0);
}
}
/// <summary>
/// Compares 2 text files and returns lines where differences occur.
/// </summary>
/// <param name="f1"></param>
/// <param name="f2"></param>
/// <returns></returns>
public static string[] Compare(TextFile f1, TextFile f2)
{
int maxIndex = Math.Min(f1.lines.Count, f2.lines.Count);
var rval = new List<string>();
if (f1.lines.Count != f2.lines.Count)
rval.Add("Length of files is different");
for (int i = 0; i < maxIndex; i++)
{
string l1 = f1.lines[i];
string l2 = f2.lines[i];
if (l1 != l2)
{
rval.Add("File 1: <" + l1+"> File 2:<"+l2+">");
}
}
return rval.ToArray();
}
public void Append(string filename)
{
TextFile tf = new TextFile(filename);
lines.AddRange(tf.lines);
}
private void ProcessFile(RouteOptions routing, string fileName)
{
string importTag = "import"; // used to make friendly export filename
try
{
TextFile tf = new TextFile(fileName);
SeriesList sl = new SeriesList();
if (HydrometInstantSeries.IsValidDMS3(tf))
{
importTag = "decodes";
sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf);
}
else if( HydrometDailySeries.IsValidArchiveFile(tf))
{
importTag = "htools";
sl = HydrometDailySeries.HydrometDailyDataToSeriesList(tf);
}
else if (LoggerNetFile.IsValidFile(tf))
{
LoggerNetFile lf = new LoggerNetFile(tf);
if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0)
{
importTag = lf.SiteName;
sl = lf.ToSeries(validPcodes);
}
}
//else if (DecodesRawFile.IsValidFile(tf))
//{
// DecodesRawFile df = new DecodesRawFile(tf);
// importTag = "raw";
// sl = df.ToSeries();
//}
else
{
Logger.WriteLine("skipped Unknown File Format: " + fileName);
return;
}
m_importer = new TimeSeriesImporter(m_db, routing,m_saveOption);
Console.WriteLine("Found " + sl.Count + " series in " + fileName);
foreach (var item in sl)
{
Logger.WriteLine(item.Table.TableName);
}
if (sl.Count > 0)
{
m_importer.Import(sl, m_computeDependencies, m_computeDailyDependencies,importTag);
FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "attic", fileName);
}
}
catch (Exception ex)
{
Logger.WriteLine("Error:" + ex.Message);
Console.WriteLine("Error: skipping file, will move to error subdirectory " + fileName);
FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "error", fileName);
}
}
public UsgsRDBFile(string[] data, bool allTextColumns=false)
{
m_allTextColumns = allTextColumns;
tf = new TextFile(data);
ParseFile();
}
public static TimeSeriesDatabaseDataSet.RatingTableDataTable GetRatingTable(string cbtt, string pcode,string ratingName)
{
string url = "";
if (HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.GreatPlains)
{
url = "https://www.usbr.gov/gp-bin/expandrtf.pl?site=pali&pcode=q&form=col";
url = url.Replace("site=pali", "site=" + cbtt.Trim());
url = url.Replace("pcode=q", "pcode=" + pcode.Trim());
}
else if( HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.PN)
{
url = "http://lrgs1.pn.usbr.gov/rating_tables/"+ratingName+".csv";
if( !NetworkUtility.Intranet)
url = "https://www.usbr.gov/pn/hydromet/configurationdata/rating_tables/" + ratingName + ".csv";
var tmp = FileUtility.GetTempFileName(".csv");
Web.GetFile(url, tmp);
var rt = new TimeSeriesDatabaseDataSet.RatingTableDataTable();
rt.ReadFile(tmp);
return rt;
}
else if (HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.Yakima)
{ // yakima ?
url = "https://www.usbr.gov/pn-bin/yak/expandrtf.pl?site=kee&pcode=af&form=col";
url = url.Replace("site=kee", "site=" + cbtt.Trim());
url = url.Replace("pcode=af", "pcode=" + pcode.Trim());
}
string[] data = Web.GetPage(url);
TextFile tf = new TextFile(data);
return ParseRatingTableData(tf, cbtt, pcode);
}
private void ReadHistoricalFilesFromWeb(DateTime t1, DateTime t2)
{
int wy = DateTime.Now.WaterYear();
TextFile tf = new TextFile(new string[] { });
if (!InCurrentWaterYear(t1, t2))
{
//string filename = Path.Combine(dir, siteName + "_all.txt");
var data = Web.GetPage(url + state + "/" + siteName + "_all.txt", true);
tf.Append(data);
}
var lines = Web.GetPage(url + state + "/" + siteName + "_" + wy + ".tab", true);
tf.Append(lines);
int errorCount = 0;
if (tf.Length == 0)
return;
string a = tf[0];
int idx = a.LastIndexOf("-");
Name = a.Substring(0, idx);
a = a.Substring(idx + 1);
string[] cols = TextFile.Split(a);
int idxData = Array.IndexOf(cols, columnName);
if (idxData < 0)
{
Console.WriteLine("Can't find column '" + columnName + "'");
return;
}
for (int i = 1; i < tf.Length; i++)
{
string[] tokens = tf[i].Split('\t');//.Split(i);
if (idxData >= tokens.Length)
{
continue;
}
if (!Regex.IsMatch(tokens[0], "[0-9]{6}"))
{
errorCount++;
if (errorCount < 100)
Console.WriteLine("skipping:'" + tf[i] + "'");
continue;
}
DateTime t = ParseDate(tokens[0]);
if (columnName.Trim().ToLower() != "prcp")
{
t = t.AddDays(-1);
}
if (t < t1 || t >= t2)
continue;
double num = 0;
if (!Double.TryParse(tokens[idxData], out num))
{
errorCount++;
if (errorCount < 100)
{
Console.WriteLine("Error parsing '" + tokens[idxData] + "'");
Console.WriteLine("from line " + tf[i]);
}
AddMissing(t);
continue;
}
Add(t, num);
}
}
public void IndexOfRegex()
{
TextFile tf = new TextFile(data);
Assert.AreEqual(9, tf.IndexOfRegex("Jackson$"));
Assert.AreEqual(4, tf.IndexOfRegex("object_name: Jackson"));
Assert.AreEqual(13, tf.IndexOfRegex("^object_name: Jackson$"));
Assert.AreEqual(16,
tf.IndexOfBothRegex("^object_name: Jackson$", "^slot_name: Inflow$"));
}
public Yakima82RatingFile(string filename)
{
tf = new TextFile(filename);
}
public void Append(string filename)
{
TextFile tf = new TextFile(filename);
lines.AddRange(tf.lines);
}
public void NthIndexOf()
{
TextFile tf = new TextFile(data);
int idx = tf.NthIndexOf("END_SLOT", 3);
Assert.AreEqual(14, idx);
}
private static bool CompareHydrometData(TextFile a, TextFile b)
{
int idx1 = a.IndexOf("BEGIN DATA");
a.DeleteLines(0,idx1);
int idx2 = a.IndexOf("END DATA");
a.DeleteLines(idx2, a.Length - 1);
idx1 = b.IndexOf("BEGIN DATA");
b.DeleteLines(0, idx1);
idx2 = b.IndexOf("END DATA");
b.DeleteLines(idx2, b.Length - 1);
if (a.Length != b.Length)
return false;
for (int i = 0; i < a.Length; i++)
{
var x = a[i].Split(',');
var y = b[i].Split(',');
if (x.Length != y.Length)
return false;
for (int j = 0; j < x.Length; j++)
{
if (i == 0) // cbtt parameter
{// allow different spacing
// 'jck fb' or 'jck fb'
RegexOptions options = RegexOptions.None;
Regex regex = new Regex("[ ]{2,}", options);
x[j] = regex.Replace(x[j], " ");
y[j] = regex.Replace(y[j], " ");
}
if (x[j].Trim() != y[j].Trim())
return false;
}
}
return true;
}
protected override void ReadCore(DateTime t1, DateTime t2)
{
// http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Achj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=HTML&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=19&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t=
//http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Aid.chj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=Text+Comma-Delimited&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=15&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t=
//record://los/hf//ir-month/drxzzazd/ time:-1m
string url = "http://www.nwd-wc.usace.army.mil/cgi-bin/dataquery.pl?k=id:[email protected]+record:";
url += m_path + "&sd=01&sm=JAN&sy=2007&ed=23&em=FEB&ey=2007&of=Text+Comma-Delimited";
url = ApplyDatesToURL(t1, t2, url);
string[] results = Reclamation.Core.Web.GetPage(url);
TextFile tf = new TextFile(results);
string msg = "<P>The web server is too loaded right now to service your request. Please wait a few minutes and try your request again. ";
int idx = tf.IndexOf(msg);
if (idx >= 0)
throw new ApplicationException(msg);
string pattern = @"(?<date>\d{1,2}[A-Za-z]{3}\d{4}),(?<hour>\d\d:\d\d),(?<value>[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?)";
Regex re = new Regex(pattern, RegexOptions.Compiled);
Console.WriteLine(pattern);
for (int i = 0; i < results.Length; i++)
{
DateTime t = DateTime.MinValue;
double val = Point.MissingValueFlag;
Match m = re.Match(results[i]);
if (m.Success)
{
GroupCollection g = m.Groups;
if (g["hour"].Value == "24:00")
{
if( DateTime.TryParse(g["date"].Value, out t)
&& double.TryParse(g["value"].Value, out val))
{
t = t.AddDays(1);
AddVal(t, val);
}
}
else
if (DateTime.TryParse(g["date"].Value+" "+g["hour"].Value, out t)
&& double.TryParse(g["value"].Value, out val))
{
AddVal(t, val);
}
else
{
SkipMessage("skipping " + results[i]);
}
}
else
{
SkipMessage("skipping " + results[i]);
}
// Console.WriteLine(results[i]);
}
}
private void AddCampbellDataLogger(object sender, EventArgs e)
{
if (openFileDialogCr10x.ShowDialog() == DialogResult.OK)
{
TextFile tf = new TextFile(openFileDialogCr10x.FileName);
if (LoggerNetFile.IsValidFile(tf))
{
// import all columns in LoggerNet file.
LoggerNetFile ln = new LoggerNetFile(tf);
foreach (var item in ln.ToSeries())
{
DB.AddSeries(item);
}
//ln.ToSeries()
}
else
{
ImportCr10x();
}
}
}
/// <summary>
/// This method generates the full table from the RDB file
/// </summary>
/// <param name="rdbFile"></param>
private void CreateFullRatingTable(TextFile rdbFile)
{
// Build container DataTable
DataTable ratingTable = new DataTable();
ratingTable.Columns.Add(new DataColumn("Stage", typeof(double)));
ratingTable.Columns.Add(new DataColumn("Shift", typeof(double)));
ratingTable.Columns.Add(new DataColumn("Flow", typeof(double)));
// Save RDB file
var tempFile = Path.GetTempFileName();
rdbFile.SaveAs(tempFile);
rdbFile = new TextFile(tempFile);
// Loop through each RDB file row to find the data
int headerRow = rdbFile.IndexOf("INDEP SHIFT DEP STOR") + 2;
for (int i = headerRow; i < rdbFile.Length; i++)
{
var row = rdbFile[i];
var newRow = ratingTable.NewRow();
var rowVals = rdbFile[i].Split('\t');
newRow["Stage"] = Convert.ToDouble(rowVals[0]);
newRow["Shift"] = Convert.ToDouble(rowVals[1]);
newRow["Flow"] = Convert.ToDouble(rowVals[2]);
ratingTable.Rows.Add(newRow);
}
this.fullRatingTable = ratingTable;
}
/// <summary>
/// This method generates the HJ and Q tables
/// </summary>
/// <param name="station"></param>
private void CreateShiftAndFlowTables(TextFile rdbFile)
{
var tempFile = Path.GetTempFileName();
rdbFile.SaveAs(tempFile);
rdbFile = new TextFile(tempFile);
var rdbFileString = rdbFile.FileContents;
var rdbItems = rdbFileString.Split('\t', '\n', '\r', ' ').ToList();//[JR] these separate the rdb file into chunks, make sure that new line characters are specified in this line
var skelPtIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i] == "*").ToList();//get skeletal points idx
var breakptIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i].Contains("BREAKPOINT")).ToList();//get breakpoint idx
var offsetIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i].Contains("OFFSET")).ToList();//get offset idx
// Generate HJ Table by defining stage-shift pairs within a C# DataTable
var hjTable = CreateHJTable(rdbFileString);
// Define Logarithmic coefficient pairs in a C# DataTable
var coeffTable = new DataTable();
coeffTable.Columns.Add(new DataColumn("breakpoint", typeof(double)));
coeffTable.Columns.Add(new DataColumn("offset", typeof(double)));
if (offsetIdx.Count == 0) // no breakpoint and coefficient data
{
var coeffRow = coeffTable.NewRow();
coeffRow["breakpoint"] = -999999999.99;
coeffRow["offset"] = 0.0;
coeffTable.Rows.Add(coeffRow);
}
else if (offsetIdx.Count == 1)
{
var coeffRow = coeffTable.NewRow();
coeffRow["breakpoint"] = -999999999.99;
coeffRow["offset"] = Convert.ToDouble(rdbItems[offsetIdx[0]].Split('=')[1].ToString().Replace("\"", ""));
coeffTable.Rows.Add(coeffRow);
}
else
{
for (int i = 0; i < offsetIdx.Count; i++)
{
var coeffRow = coeffTable.NewRow();
if (i == 0)
{ coeffRow["breakpoint"] = -999999999.99; }
else
{ coeffRow["breakpoint"] = Convert.ToDouble(rdbItems[breakptIdx[i - 1]].Split('=')[1].ToString().Replace("\"", "")); }
coeffRow["offset"] = Convert.ToDouble(rdbItems[offsetIdx[i]].Split('=')[1].ToString().Replace("\"", ""));
coeffTable.Rows.Add(coeffRow);
}
}
// Generate Q DataTable
var qTable = new DataTable();
qTable.Columns.Add(new DataColumn("Stage", typeof(double)));
qTable.Columns.Add(new DataColumn("Flow", typeof(double)));
qTable.Columns.Add(new DataColumn("A-Coeff", typeof(double)));
qTable.Columns.Add(new DataColumn("B-Coeff", typeof(double)));
foreach (var item in skelPtIdx)
{
var qRow = qTable.NewRow();
double ghVal = Convert.ToDouble(rdbItems[item - 3]);
double shiftVal = Convert.ToDouble(rdbItems[item - 2]);
double stageVal = ghVal + shiftVal;
qRow["Stage"] = stageVal;
qRow["Flow"] = Convert.ToDouble(rdbItems[item - 1]);
DataRow[] coeffMatch = coeffTable.Select("[breakpoint] <= '" + stageVal + "'");
qRow["A-Coeff"] = coeffMatch[coeffMatch.Count() - 1][1];
qRow["B-Coeff"] = 0.0;
qTable.Rows.Add(qRow);
}
if (qTable.Rows.Count < 1)
{ throw new Exception("No skeletal points found for station: " + this.stationName); }
this.hjTable = hjTable;
this.qTable = qTable;
}
/**********************************
*
* Create a tree file that is usable by pisces
*
* input : riverware rdf file
* output : comma seperated tree file.
*
*
* // example portion of input file.
* (this input file is using snapshots in riverware)
* -----------------------------------
* object_type: SnapShotObj
* object_name: Most Likely 2
* slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow
* END_SLOT_PREAMBLE
* units: cfs
*
* // example output. There is no nesting of tree levels for now.
* RiverwareName,Description,RiverwareDataType,Level,Units
* Riverware Results,,,0,
* Yakima River at Parker PARW,Yakima River at Parker PARW,Gage Outflow,1,cfs
* Yakima River at Grandview,Yakima River at Grandview,Gage Inflow,1,cfs
* ...
*
**********************************/
public static void AddRiverWareFileToDatabase(string rdfFilename, PiscesFolder parent,
TimeSeriesDatabase db)
{
Reclamation.Core.TextFile tf = new Reclamation.Core.TextFile(rdfFilename);
#region notes
/*
* SnapShotStyle...
* -------------------------------------
* 2001-9-29 24:00
* 2001-9-30 24:00
* object_type: SnapShotObj
* object_name: Most Likely 2 ### scernario name
* slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow # object_name slotName are combined.
* END_SLOT_PREAMBLE
* units: cfs
*
* Regular Style ...
* ---------------------------------------
* END_COLUMN
* END_SLOT
* object_type: StreamGage
* object_name: Yakima 202_0 at Easton EASW
* slot_name: Gage Outflow
* END_SLOT_PREAMBLE
* units: cfs
* scale: 1
*
*/
#endregion
int number_of_runs = LookupNumberOfRuns(tf);
PiscesFolder folder = parent;
if (number_of_runs == 1)
{
folder = db.AddFolder(parent, Path.GetFileNameWithoutExtension(rdfFilename));
}
int sz = tf.Length;
// object_type and object_name should occur on consecutive lines.
int index = tf.IndexOfBoth("object_name:", "slot_name:", 0);
var objectList = new List <string>(); //list to avoid duplicates in tree
Performance p1 = new Performance();
Performance p2 = new Performance();
p2.Pause();
int counter = 0;
db.SuspendTreeUpdates();
var sc = db.GetSeriesCatalog();
Dictionary <string, int> objTypeID = new Dictionary <string, int>();
Dictionary <string, int> objNameID = new Dictionary <string, int>();
while (index < sz && index > 0)
{
//slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow
string slot_name = tf[index + 1].Substring(11); //Andrews Gage 12447390 at RM 3_5_Gage Outflow
string object_type = tf[index - 1].Substring(13);
string object_name = tf[index].Substring(13);
string units = tf[index + 3].Substring(6).Trim();
string tag = object_name + ":" + slot_name;
if (!objectList.Contains(tag))
{
int scenarioNumber = -1;
if (number_of_runs > 1)
{
scenarioNumber = 1;
}
RiverWareSeries s;
if (object_type == "SnapShotObj")
{
s = new RiverWareSeries(rdfFilename, "", slot_name, scenarioNumber, true, units);
}
else
{
s = new RiverWareSeries(rdfFilename, object_name, slot_name, scenarioNumber, false, units);
}
s.Units = units;
s.ConnectionString = ConnectionStringUtility.MakeFileNameRelative(s.ConnectionString, db.DataSource);
p2.Continue();
if (object_type.Contains("Reservoir"))
{
object_type = "Reservoir";
}
else if (object_type.Contains("Reach"))
{
object_type = "Reach";
}
else if (object_type.Contains("Diversion"))
{
object_type = "Diversion";
}
else if (object_type.Contains("Canal"))
{
object_type = "Canal";
}
int id = sc.NextID();
if (!sc.FolderExists(object_type, folder.ID))
{
objTypeID.Add(object_type, id);
sc.AddFolder(object_type, id, folder.ID);
id++;
}
if (!sc.FolderExists(object_name, objTypeID[object_type]))
{
objNameID.Add(object_name, id);
sc.AddFolder(object_name, id, objTypeID[object_type]);
id++;
}
sc.AddSeriesCatalogRow(s, id, objNameID[object_name], "");
objectList.Add(tag);
}
index = tf.IndexOfBoth("object_name:", "slot_name:", index + 2);
counter++;
}
p1.Report("total");
p2.Report("db.add()");
//398.7732813 seconds elapsed. total
//384.6792607 seconds elapsed. db.add()
// disable tree refresh (doubles perf)
// 255.9736646 seconds elapsed. total
// 241.7702669 seconds elapsed. db.add()
// implemented member ExternalDataSource
//34.8756696 seconds elapsed. total
//20.3753912 seconds elapsed. db.add()
var convention = Reclamation.TimeSeries.RiverWare.ImportRiverWare.ScenarioConvention.Default;
if (number_of_runs > 1) // Multiple runs.
{ // show dialog to allow water year naming or traces
var dlg = new Reclamation.TimeSeries.RiverWare.ImportRiverWare();
if (dlg.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
convention = dlg.NamingConvention;
}
// Add Scenarios.
var tblScen = db.GetScenarios();
for (int i = 0; i < number_of_runs; i++)
{
string name = "Run" + i;
if (convention == RiverWare.ImportRiverWare.ScenarioConvention.ByYear)
{
name = (dlg.FirstYear + i).ToString();
}
//string scenarioPath = ConnectionStringUtility.MakeFileNameRelative("FileName=" + item, DB.Filename);
tblScen.AddScenarioRow(name, true, "ScenarioNumber=" + (i + 1).ToString());
}
db.Server.SaveTable(tblScen);
}
db.Server.SaveTable(sc);
db.ResumeTreeUpdates();
db.RefreshFolder(parent);
}
public void WaterYearTrace()
{
string path = Globals.TestDataPath;
List<string> args = new List<string>();
//-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919
args.Add(path + "\\RiverWare\\snakeControl.txt");
args.Add("c:\\temp");
args.Add("1927-10-04");// start date for RiverWare
args.Add("12:00");
args.Add("1928-07-01");
args.Add("12:00");
args.Add("1DAY");
args.Add("-UXlsFileName=" +path + "\\RiverWare\\SnakeTestData.xls");
args.Add("-STrace=4");
Reclamation.RiverwareDmi.Program.Main(args.ToArray());
string fn = @"c:\temp\Local Inflow.MinidokaToMilner_Milner.txt";
TextFile tf = new TextFile(fn);
File.Delete(fn);
int idx = tf.IndexOf("start_date");
Assert.AreEqual(-2906, Convert.ToDouble(tf[idx + 1]), .6);
Assert.AreEqual(-2864, Convert.ToDouble(tf[idx + 2]), .6);
}
/// <summary>
/// Reads Raw RBMS Text file. Adds DateTime and Value Columns
/// </summary>
/// <param name="filename"></param>
/// <returns></returns>
private static DataTable ReadRBMSFile(string filename)
{
TextFile tf = new TextFile(filename);
int fieldCount = 0;
if (tf.FileData.Length > 0)
{
fieldCount = tf[0].Split(',').Length;
if (fieldCount < 4 || fieldCount > 5)
{
throw new Exception("Error: " + filename + "is not a valid RBMS File. It should have 4 or 5 columns of data");
}
}
else
{
throw new Exception("Error: There was not data in the input file");
}
tf = null;
CsvFile raw = null;
if (fieldCount == 5)
{
string[] dataTypes = new string[] { "String", "String", "String", "String", "String" };
string[] fieldNames = new string[] { "DH", "Riser", "Date", "Time", "Measured" };
//DH-438-RS , 1,01-apr-2000,00:00:54 , 870.290
//string pattern = @"\s*[A-Za-z\-0-9]{3,}\s*,\s*\d+,\d{1,2}-[A-Za-z]{3}-\d{4},\d{2}:\d{2}:\d{2}\s*,\s*[0-9\.\-\+Ee]+";
raw = new CsvFile(filename, dataTypes, fieldNames);
}
else
{
string[] dataTypes = new string[] { "String", "String", "String", "String" };
string[] fieldNames = new string[] { "DH", "Riser", "Date", "Measured" };
//DH-259-RS , 1,01-apr-1991 00:00:32 , 866.960
//string pattern = @"\s*[A-Za-z\-0-9]{3,}\s*,\s*\d+,\d{1,2}-[A-Za-z]{3}-\d{4}\s\d{2}:\d{2}(:\d{2})?\s*,\s*[0-9\.\-\+Ee]+";
raw = new CsvFile(filename, dataTypes, fieldNames);
}
return CreateDateValueTableForDatabase(raw);
// format with date time together.
//DH-259-RS , 1,01-apr-1991 00:00:32 , 866.960
// format with date time csv.
//DH-438-RS , 1,01-apr-2000,00:00:54 , 870.290
//Console.WriteLine(db.Rows.Count);
// return raw;
}
public TextFileCredentials(string filename)
{
m_fileName = filename;
tf = new TextFile(m_fileName);
}
public static TimeSeriesDatabaseDataSet.RatingTableDataTable ParseRatingTableData(TextFile tf,
string cbtt, string pcode)
{
/*
<table border="1" summary="Expanded Rating Table"> <tr align="center">
<th width="80px"><b>ft</b></th>
<th width="80px"><b>cfs</b></th>
</tr>
*
* Table Edit Date: 18-SEP-2014 02:22
<tr align="right"><td>2440.01</td><td>-704.00</td></tr>
<tr align="right"><td>2440.02</td><td>-704.00</td></tr>
<tr align="right"><td>2440.03</td><td>-704.00</td></tr>
<tr align="right"><td>2440.04</td><td>-704.00</td></tr>
*/
TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable();
int idx = tf.IndexOfBothRegex("<th.*</th>", "<th.*</th>");
if (idx >= 0)
{
t.XUnits = Regex.Match(tf[idx], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value;
t.YUnits = Regex.Match(tf[idx + 1], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value;
}
//<br>Table Edit Date: 18-SEP-2014 02:22<br>Today's Date: 23-SEP-2014 06:54
var idxDate = tf.IndexOfRegex("Table Edit Date:.*<br>");
if (idxDate >= 0)
{
t.EditDate = Regex.Match(tf[idxDate], "Table Edit Date:(?<date>.*)<br>").Groups[1].Value;
}
Regex re = new Regex(@"<tr.*?>(?<x>[\d\.\-\+]{1,12})</td><td>(?<y>[\d\.\-\+]{1,12})</td>");
for (int i = idx + 2; i < tf.Length; i++)
{
var m = re.Match(tf[i]);
if (m.Success)
{
double x, y;
if (double.TryParse(m.Groups["x"].Value, out x)
)
{
if (double.TryParse(m.Groups["y"].Value, out y)
&& System.Math.Abs(998877 - y) > 0.01
)
{
t.AddRatingTableRow(x, y);
}
}
}
}
t.Name = "Hydromet " + cbtt + " " + pcode;
idx = tf.IndexOfRegex("<b>Station .+<br />");
if (idx >= 0)
t.Name = Regex.Match(tf[idx], "<b>Station (.+)<br />").Groups[1].Value;
return t;
}
public UsgsRDBFile(string filename)
{
tf = new TextFile(filename);
ParseFile();
}
private void ReadSnowDepth(DateTime t1, DateTime t2)
{
/*
--------------------------------------------------------------------------------
Change In
Snow Water Snow Snow Water Change In
Site Name Date Time Equivalent Depth Equivalent Snow Depth
------------------------------------------------------------------------------
ATLANTA SUMMIT 01/07 0000 16.1 58.0
01/08 0000 16.1 57.0 0.0 -1.0
01/09 0000 16.0 57.0 -0.1 0.0
01/10 0000 15.9 57.0 -0.1 0.0
01/11 0000 15.9 56.0 0.0 -1.0
01/12 0000 16.0 56.0 0.1 0.0
01/13 0000 16.8 61.0 0.8 5.0
01/14 0000 17.4 64.0 0.6 3.0
*
--------------------------------------------------------------------------------
Change In
Snow Water Snow Snow Water Change In
Site Name Date Time Equivalent Depth Equivalent Snow Depth
------------------------------------------------------------------------------
JACKSON PEAK 01/07 0000 13.5 50.0
01/08 0000 13.5 48.0 0.0 -2.0
01/09 0000 13.4 49.0 -0.1 1.0
01/10 0000 13.4 48.0 0.0 -1.0
01/11 0000 13.4 48.0 0.0 0.0
01/12 0000 13.5 -99.9 0.1 -99.9
01/13 0000 14.5 -99.9 1.0 -99.9
01/14 0000 15.4 62.0 0.9 -99.9
------------------------------------------------------------------------------
------------------------------------------------------------------------------
*/
var data = Web.GetPage(urlDepth + state + "/" + siteName + ".txt", true);
TextFile tf = new TextFile(data);
var idx = tf.IndexOf("Site Name Date Time Equivalent Depth");
if (idx > 0)
{
string pattern = "^.{23}(?<month>\\d{2})/(?<day>\\d{2})\\s{1}0000\\s*(?<se>[0-9\\.+-]+)\\s*(?<sd>[0-9\\.+-]+)";
for (int i = idx+2; i < idx+10; i++)
{
Regex re = new Regex(pattern);
var m = re.Match(tf[i]);
if (m.Success)
{
int month = int.Parse(m.Groups["month"].Value);
int day = int.Parse(m.Groups["day"].Value);
double sd = double.Parse(m.Groups["sd"].Value);
DateTime t = new DateTime(DateTime.Now.Year, month, day);
if (t >= t1 && t <= t2)
{
if (System.Math.Abs(sd - 99.9) < 0.1)
AddMissing(t);
Add(t, sd);
}
}
}
}
}
public TextFileCredentials(string filename)
{
m_fileName = filename;
tf = new TextFile(m_fileName);
}
