public void WaterYear1931() { Performance p = new Performance(); string path = Globals.TestDataPath; //controlfile tempPath yyyy-mm-dd hh:mm yyyy-mm-dd hh:mm 1DAY -UxlsFileName=file.xls [-STrace=n]"); //-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919 List<string> args = new List<string>(); args.Add(path + "\\RiverWare\\snakeControl.txt"); args.Add("c:\\temp"); args.Add("1927-10-04");// start date for RiverWare args.Add("12:00"); args.Add("1928-07-01");// end date for riverware args.Add("12:00"); args.Add("1DAY"); args.Add("-UXlsFileName=" + path + "\\Riverware\\SnakeTestData.xls"); args.Add("-UWaterYear=1931"); // actual xls data begins 10/4/1930 (water year 1931) args.Add("-UFirstWaterYear=1928"); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = @"c:\temp\Inflow.Jackson.txt"; TextFile tf = new TextFile(fn); File.Delete(fn); Assert.AreEqual("# import began on line index: 1113", tf[tf.IndexOf("# import")]); Assert.AreEqual("start_date: 1927-10-04 24:00", tf[tf.IndexOf("start_date")]); int idx = tf.IndexOf("start_date"); Assert.AreEqual(691.4, Convert.ToDouble(tf[idx+1]),.1); Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 2]), .1); Assert.AreEqual(689.4, Convert.ToDouble(tf[idx + 3]), .1); Assert.AreEqual(4211.4, Convert.ToDouble(tf[idx + 4]), .1); p.Report("completed FullDataDump()"); }
public static TimeSeriesDatabaseDataSet.RatingTableDataTable GetRatingTable(string siteNumber) { //string url = "http://waterdata.usgs.gov/nwisweb/data/exsa_rat/13236500.rdb"; string url = "http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=13081500&file_type=exsa"; url = url.Replace("13081500", siteNumber); string[] data = Web.GetPage(url); TextFile tf = new TextFile(data); UsgsRDBFile rdb = new UsgsRDBFile(data); TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable(); t.XUnits = LookupUnits(tf,"# //RATING_INDEP"); t.YUnits = LookupUnits(tf,"# //RATING_DEP"); for (int i = 0; i < rdb.Rows.Count; i++) { double x = Convert.ToDouble(rdb.Rows[i]["indep"]); double y = Convert.ToDouble(rdb.Rows[i]["dep"]); t.AddRatingTableRow(x, y); } t.Name = "Usgs " + siteNumber; return t; }
public DataQueryInfo(HecDssPath path) { string url = "http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=" + path.B; results = Reclamation.Core.Web.GetPage(url); tf = new TextFile(results); this.path = path; name = GetName(); ParseAttributes(); }
private static string ConvertCSVToShiftFormat(string html, string[] cbtt) { /* * DATE , GAGE HT , CFS , SHIFT , 4/18/2015 , 1.04 , 152.16 , +0.29 , 4/21/2015 , 1.07 , 151.5 , +0.27 , 5/15/2015 , 1.56 , 211.26 , +0.11 , 6/16/2015 , 1.77 , 118.36 , -0.64 , 7/25/2015 , 1.67 , 151.3 , -0.35 , , , , , , , , , */ string[] CRLF = { "\r\n" }; string[] lines = html.Split(CRLF, StringSplitOptions.None); string cleanFile = "cbtt,pcode,date_measured,discharge,stage,shift\r\n"; var tf = new TextFile(html.Split(new char[]{'\n','\r'}, StringSplitOptions.RemoveEmptyEntries)); for (int i = 0; i < cbtt.Length; i++) { var idx = tf.IndexOf(cbtt[i]); if( idx >=0) { var idxDate = tf.IndexOf("DATE", idx); if( idxDate > idx+5)// date should be within 5 lines of cbtt { Console.WriteLine("Error: did not find DATE with cbtt ="+cbtt); continue; } // now parse data until it runs out for (int j = idxDate+1; j < tf.Length; j++) { DateTime t; var tokens = tf[j].Split(','); if( tokens.Length < 4) break; if( !DateTime.TryParseExact(tokens[0].Trim(), "M/d/yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out t) ) break; var x = cbtt[i] + ",CH," + t.ToShortDateString() + "," + tokens[2].Trim() + "," + tokens[1].Trim() + "," + tokens[3].Trim(); cleanFile += x + "\r\n"; Console.WriteLine(x); } } else { Console.WriteLine("Error: did not find "+cbtt[i]); } } return cleanFile; }
/// <summary> /// Reads shef .A file and extracts all shef locations and codes that match /// simple, only reads single parameter per line /// </summary> /// <param name="filename">input file to read</param> /// <param name="shefloc">shef location to search for</param> /// <param name="shefCode">shefcode to search for</param> /// <returns></returns> public static Series ReadSimpleShefA(string filename, string shefloc, string shefCode) { //.A BRWLD 150309 PD DH2315 /HE1H .00 //.A BRWLD 150309 PD DH2315 /HE2H .19 var rval = new Series(); TextFile tf = new TextFile(filename); for (int i = 0; i < tf.Length; i++) { var m = ShefASimpleRegex.Match(tf[i]); if (m.Success) { var loc = m.Groups["shefloc"].Value; var sc = m.Groups["shefcode"].Value; var yy = m.Groups["year"].Value; var mm = m.Groups["month"].Value; var dd = m.Groups["day"].Value; var hh = m.Groups["hour"].Value; var min = m.Groups["min"].Value; var val = m.Groups["value"].Value; if (shefCode == sc && loc == shefloc) { int year, month, day, hour,minute; double num = Point.MissingValueFlag; ; if (int.TryParse(yy, out year) && int.TryParse(mm, out month) && int.TryParse(dd, out day) && int.TryParse(hh, out hour) && int.TryParse(min, out minute) && double.TryParse(val,out num)) { var t = new DateTime(year + 2000, month, day, hour, minute,0); if( rval.IndexOf(t)>=0) { Console.WriteLine("Skipping duplicate "+t.ToString()+" :" +num); continue; } rval.Add(t, num); } } } } return rval; }
/*************************************** * returns list of dates from beginning of riverware file in *.rdf format. **************************************************/ private List <DateTime> ReadDates(Reclamation.Core.TextFile tf, int startingIndex) { List <DateTime> rval = new List <DateTime>(); int idx = tf.IndexOfAny(new string[] { "time_steps:", "timesteps" }, startingIndex); if (idx < 0) { throw new Exception("Error looking for 'timesteps' or 'time_steps:' in " + tf.FileName); } //timesteps:15706 //time_steps:366 string s = tf[idx]; int i2 = s.IndexOf(":"); s = s.Substring(i2 + 1); // just the number part int numDates = Convert.ToInt32(s); int idxToDates = tf.IndexOf("END_RUN_PREAMBLE", idx); if (idxToDates < 0) { throw new Exception("Error looking for END_RUN_PREAMBLE in " + tf.FileName); } idxToDates++; for (int i = 0; i < numDates; i++) { DateTime d; string dateString = tf[idxToDates + i]; dateString = dateString.Replace(" 24:00", " 23:59:59"); if (DateTime.TryParse(dateString, out d)) { rval.Add(d); } else { throw new Exception("Can't parse " + tf[idxToDates + i] + " as a date and time"); } } return(rval); }
/// <summary> /// Compares data in two usgs rdb files, ignoring comments at the beginning /// </summary> /// <param name="tf1"></param> /// <param name="tf2"></param> /// <returns>true if there are differences in the data</returns> public static bool Diff(TextFile tf1, TextFile tf2) { var diff = System.Math.Abs(tf1.Length - tf2.Length); if ( diff > 5) { Logger.WriteLine("difference in length of files: " + diff); return true; //different number of lines in file } var idx1 = tf1.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR"); var idx2 = tf2.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR"); tf1.DeleteLines(0, idx1); tf2.DeleteLines(0, idx2); return TextFile.Compare(tf1, tf2).Length != 0; }
private static int LookupNumberOfRuns(Reclamation.Core.TextFile tf) { int number_of_runs = 1; //:53 //number_of_runs:53 int index = tf.IndexOf("number_of_runs:"); if (index == -1) { Logger.WriteLine("Error: number_of_runs is not defined in file " + tf.FileName); } else { number_of_runs = Convert.ToInt32(tf[index].Substring(15)); } return(number_of_runs); }
public void CGI_Daily_CleanerDesign() { string payload = "parameter=luc fb,luc af&start=2016-5-19&end=2016-5-24&format=csv"; var fn = RunTest(payload, TimeInterval.Daily); /* DATE , LUC FB , LUC AF 5/24/2016 4:10:00 PM: CreateSQL 5/24/2016 4:10:00 PM: list of 2 series 05/19/2016,, 253349.11 05/20/2016,, 253698.69 05/21/2016,, 254910.41 05/22/2016,, 255826.27 05/23/2016,, 255933.44 */ TextFile tf = new TextFile(fn); int idx = tf.IndexBeginningWith("05/19/2016", 0); var s = tf[idx].Split(',')[2]; double d = Convert.ToDouble(s); Assert.AreEqual(253349.11, d, 1.0); }
public void MultiYearTest() { Performance p = new Performance(); string path = Reclamation.Core.Globals.TestDataPath; for (int i = 1; i <= 2; i++) { string dir = @"C:\Temp\dmitest"; Directory.CreateDirectory(dir); List<string> args = new List<string>(); args.Add(path + "\\RiverWare\\boiseControl.txt"); args.Add(dir); args.Add("1927-10-04");// date for RiverWare args.Add("12:00"); args.Add("1928-07-01"); args.Add("12:00"); args.Add("-UXlsFileName=" + path + "\\RiverWare\\BoiseModelData.xls"); args.Add("-sTrace="+i); if( Directory.Exists(dir)) Directory.Delete(dir,true); Directory.CreateDirectory(dir); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = dir + "\\Inflow.Anderson Ranch.txt"; TextFile tf = new TextFile(fn); int idx = tf.IndexOf("start_date"); if (i == 1) // 1918 { double val = Convert.ToDouble(tf[idx + 1]); Assert.AreEqual(429, val, .5); } if (i == 2) // 1919 { double val = Convert.ToDouble(tf[idx + 1]); Assert.AreEqual(348, val, .5); } Directory.Delete(dir, true); } }
/// <summary> /// Main constructor for this class /// </summary> /// <param name="idNumber"></param> public UsgsRatingTable(string idNumber) { this.idNumber = idNumber; // Get and assign RDB file from the web string nwisURL = "http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=XXXXXXXX&file_type=exsa"; downloadURL = nwisURL.Replace("XXXXXXXX", idNumber); var newData = Web.GetPage(nwisURL.Replace("XXXXXXXX", idNumber)); if (newData.Count() == 0) { throw new Exception("NWIS data not found. Check inputs or retry later."); } webRdbTable = new TextFile(newData); webRdbTable.DeleteLine(webRdbTable.Length - 1); //last line from web is blank and the exisitng RDB does not have an empty last line // Get and assign RDB file properties this.units = webRdbTable.ReadString("LABEL=").Replace("\"", ""); this.stationName = webRdbTable.ReadString("STATION NAME=").Replace("\"", ""); this.timeZone = webRdbTable.ReadString("TIME_ZONE=").Replace("\"", ""); this.ratingTableVersion = webRdbTable.ReadString("RATING ID=").Replace("\"", ""); this.ratingTableComments = webRdbTable.ReadString("COMMENT=").Replace("\"", ""); this.ratingTableExpansion = webRdbTable.ReadString("RATING EXPANSION=").Replace("\"", ""); }
public static void CompareLinuxToVMSCGI(string payload,TimeInterval interval= TimeInterval.Irregular) { //Program.Main(new string[] { "--cgi=instant", "--payload=?"+payload }); TimeSeriesDatabase db = TimeSeriesDatabase.InitDatabase(new Arguments(new string[]{})); WebTimeSeriesWriter c = new WebTimeSeriesWriter(db,interval,payload); var fn = FileUtility.GetTempFileName(".txt"); Console.WriteLine("linux temp file:"+fn); c.Run(fn); TextFile tf = new TextFile(fn); tf.DeleteLines(0, 1); var fnhyd0 = FileUtility.GetTempFileName(".txt"); Console.WriteLine("vms temp file:" + fnhyd0); string url = "https://www.usbr.gov/pn-bin/webarccsv.pl?"; if( interval == TimeInterval.Irregular || interval == TimeInterval.Hourly) url = "https://www.usbr.gov/pn-bin/webdaycsv.pl?"; Web.GetFile(url + payload, fnhyd0); var tf2 = new TextFile(fnhyd0); if (!CompareHydrometData(tf, tf2)) { // do detailed comparision. var diff = TextFile.Compare(tf, tf2); if (diff.Length > 0) { for (int i = 0; i < tf.Length; i++) { Console.WriteLine(tf[i]); } } Assert.IsTrue(diff.Length == 0); } }
/// <summary> /// Compares 2 text files and returns lines where differences occur. /// </summary> /// <param name="f1"></param> /// <param name="f2"></param> /// <returns></returns> public static string[] Compare(TextFile f1, TextFile f2) { int maxIndex = Math.Min(f1.lines.Count, f2.lines.Count); var rval = new List<string>(); if (f1.lines.Count != f2.lines.Count) rval.Add("Length of files is different"); for (int i = 0; i < maxIndex; i++) { string l1 = f1.lines[i]; string l2 = f2.lines[i]; if (l1 != l2) { rval.Add("File 1: <" + l1+"> File 2:<"+l2+">"); } } return rval.ToArray(); }
public void Append(string filename) { TextFile tf = new TextFile(filename); lines.AddRange(tf.lines); }
private void ProcessFile(RouteOptions routing, string fileName) { string importTag = "import"; // used to make friendly export filename try { TextFile tf = new TextFile(fileName); SeriesList sl = new SeriesList(); if (HydrometInstantSeries.IsValidDMS3(tf)) { importTag = "decodes"; sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf); } else if( HydrometDailySeries.IsValidArchiveFile(tf)) { importTag = "htools"; sl = HydrometDailySeries.HydrometDailyDataToSeriesList(tf); } else if (LoggerNetFile.IsValidFile(tf)) { LoggerNetFile lf = new LoggerNetFile(tf); if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0) { importTag = lf.SiteName; sl = lf.ToSeries(validPcodes); } } //else if (DecodesRawFile.IsValidFile(tf)) //{ // DecodesRawFile df = new DecodesRawFile(tf); // importTag = "raw"; // sl = df.ToSeries(); //} else { Logger.WriteLine("skipped Unknown File Format: " + fileName); return; } m_importer = new TimeSeriesImporter(m_db, routing,m_saveOption); Console.WriteLine("Found " + sl.Count + " series in " + fileName); foreach (var item in sl) { Logger.WriteLine(item.Table.TableName); } if (sl.Count > 0) { m_importer.Import(sl, m_computeDependencies, m_computeDailyDependencies,importTag); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "attic", fileName); } } catch (Exception ex) { Logger.WriteLine("Error:" + ex.Message); Console.WriteLine("Error: skipping file, will move to error subdirectory " + fileName); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "error", fileName); } }
public UsgsRDBFile(string[] data, bool allTextColumns=false) { m_allTextColumns = allTextColumns; tf = new TextFile(data); ParseFile(); }
public static TimeSeriesDatabaseDataSet.RatingTableDataTable GetRatingTable(string cbtt, string pcode,string ratingName) { string url = ""; if (HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.GreatPlains) { url = "https://www.usbr.gov/gp-bin/expandrtf.pl?site=pali&pcode=q&form=col"; url = url.Replace("site=pali", "site=" + cbtt.Trim()); url = url.Replace("pcode=q", "pcode=" + pcode.Trim()); } else if( HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.PN) { url = "http://lrgs1.pn.usbr.gov/rating_tables/"+ratingName+".csv"; if( !NetworkUtility.Intranet) url = "https://www.usbr.gov/pn/hydromet/configurationdata/rating_tables/" + ratingName + ".csv"; var tmp = FileUtility.GetTempFileName(".csv"); Web.GetFile(url, tmp); var rt = new TimeSeriesDatabaseDataSet.RatingTableDataTable(); rt.ReadFile(tmp); return rt; } else if (HydrometInfoUtility.HydrometServerFromPreferences() == HydrometHost.Yakima) { // yakima ? url = "https://www.usbr.gov/pn-bin/yak/expandrtf.pl?site=kee&pcode=af&form=col"; url = url.Replace("site=kee", "site=" + cbtt.Trim()); url = url.Replace("pcode=af", "pcode=" + pcode.Trim()); } string[] data = Web.GetPage(url); TextFile tf = new TextFile(data); return ParseRatingTableData(tf, cbtt, pcode); }
private void ReadHistoricalFilesFromWeb(DateTime t1, DateTime t2) { int wy = DateTime.Now.WaterYear(); TextFile tf = new TextFile(new string[] { }); if (!InCurrentWaterYear(t1, t2)) { //string filename = Path.Combine(dir, siteName + "_all.txt"); var data = Web.GetPage(url + state + "/" + siteName + "_all.txt", true); tf.Append(data); } var lines = Web.GetPage(url + state + "/" + siteName + "_" + wy + ".tab", true); tf.Append(lines); int errorCount = 0; if (tf.Length == 0) return; string a = tf[0]; int idx = a.LastIndexOf("-"); Name = a.Substring(0, idx); a = a.Substring(idx + 1); string[] cols = TextFile.Split(a); int idxData = Array.IndexOf(cols, columnName); if (idxData < 0) { Console.WriteLine("Can't find column '" + columnName + "'"); return; } for (int i = 1; i < tf.Length; i++) { string[] tokens = tf[i].Split('\t');//.Split(i); if (idxData >= tokens.Length) { continue; } if (!Regex.IsMatch(tokens[0], "[0-9]{6}")) { errorCount++; if (errorCount < 100) Console.WriteLine("skipping:'" + tf[i] + "'"); continue; } DateTime t = ParseDate(tokens[0]); if (columnName.Trim().ToLower() != "prcp") { t = t.AddDays(-1); } if (t < t1 || t >= t2) continue; double num = 0; if (!Double.TryParse(tokens[idxData], out num)) { errorCount++; if (errorCount < 100) { Console.WriteLine("Error parsing '" + tokens[idxData] + "'"); Console.WriteLine("from line " + tf[i]); } AddMissing(t); continue; } Add(t, num); } }
public void IndexOfRegex() { TextFile tf = new TextFile(data); Assert.AreEqual(9, tf.IndexOfRegex("Jackson$")); Assert.AreEqual(4, tf.IndexOfRegex("object_name: Jackson")); Assert.AreEqual(13, tf.IndexOfRegex("^object_name: Jackson$")); Assert.AreEqual(16, tf.IndexOfBothRegex("^object_name: Jackson$", "^slot_name: Inflow$")); }
public Yakima82RatingFile(string filename) { tf = new TextFile(filename); }
public void NthIndexOf() { TextFile tf = new TextFile(data); int idx = tf.NthIndexOf("END_SLOT", 3); Assert.AreEqual(14, idx); }
private static bool CompareHydrometData(TextFile a, TextFile b) { int idx1 = a.IndexOf("BEGIN DATA"); a.DeleteLines(0,idx1); int idx2 = a.IndexOf("END DATA"); a.DeleteLines(idx2, a.Length - 1); idx1 = b.IndexOf("BEGIN DATA"); b.DeleteLines(0, idx1); idx2 = b.IndexOf("END DATA"); b.DeleteLines(idx2, b.Length - 1); if (a.Length != b.Length) return false; for (int i = 0; i < a.Length; i++) { var x = a[i].Split(','); var y = b[i].Split(','); if (x.Length != y.Length) return false; for (int j = 0; j < x.Length; j++) { if (i == 0) // cbtt parameter {// allow different spacing // 'jck fb' or 'jck fb' RegexOptions options = RegexOptions.None; Regex regex = new Regex("[ ]{2,}", options); x[j] = regex.Replace(x[j], " "); y[j] = regex.Replace(y[j], " "); } if (x[j].Trim() != y[j].Trim()) return false; } } return true; }
protected override void ReadCore(DateTime t1, DateTime t2) { // http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Achj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=HTML&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=19&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t= //http://www.nwd-wc.usace.army.mil/perl/dataquery.pl?k=id%3ACHJ%2Brecord%3A%2F%2FCHJ%2FYT%2F%2FIR-MONTH%2FIRVZZBZD%2F%2Bpk%3Aid.chj&sd=23&sm=MAY&sy=2011&ed=24&em=MAY&ey=2011&of=Text+Comma-Delimited&et=Screen&dc=One+Column&snpt=Daily&snph=00&snpw=15&f1m=MAY&f1d=24&f2=on&f2sm=MAY&f2sd=15&f2em=MAY&f2ed=24&f3c=Less+Than+Or+Equal+To&f3t= //record://los/hf//ir-month/drxzzazd/ time:-1m string url = "http://www.nwd-wc.usace.army.mil/cgi-bin/dataquery.pl?k=id:[email protected]+record:"; url += m_path + "&sd=01&sm=JAN&sy=2007&ed=23&em=FEB&ey=2007&of=Text+Comma-Delimited"; url = ApplyDatesToURL(t1, t2, url); string[] results = Reclamation.Core.Web.GetPage(url); TextFile tf = new TextFile(results); string msg = "<P>The web server is too loaded right now to service your request. Please wait a few minutes and try your request again. "; int idx = tf.IndexOf(msg); if (idx >= 0) throw new ApplicationException(msg); string pattern = @"(?<date>\d{1,2}[A-Za-z]{3}\d{4}),(?<hour>\d\d:\d\d),(?<value>[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?)"; Regex re = new Regex(pattern, RegexOptions.Compiled); Console.WriteLine(pattern); for (int i = 0; i < results.Length; i++) { DateTime t = DateTime.MinValue; double val = Point.MissingValueFlag; Match m = re.Match(results[i]); if (m.Success) { GroupCollection g = m.Groups; if (g["hour"].Value == "24:00") { if( DateTime.TryParse(g["date"].Value, out t) && double.TryParse(g["value"].Value, out val)) { t = t.AddDays(1); AddVal(t, val); } } else if (DateTime.TryParse(g["date"].Value+" "+g["hour"].Value, out t) && double.TryParse(g["value"].Value, out val)) { AddVal(t, val); } else { SkipMessage("skipping " + results[i]); } } else { SkipMessage("skipping " + results[i]); } // Console.WriteLine(results[i]); } }
private void AddCampbellDataLogger(object sender, EventArgs e) { if (openFileDialogCr10x.ShowDialog() == DialogResult.OK) { TextFile tf = new TextFile(openFileDialogCr10x.FileName); if (LoggerNetFile.IsValidFile(tf)) { // import all columns in LoggerNet file. LoggerNetFile ln = new LoggerNetFile(tf); foreach (var item in ln.ToSeries()) { DB.AddSeries(item); } //ln.ToSeries() } else { ImportCr10x(); } } }
/// <summary> /// This method generates the full table from the RDB file /// </summary> /// <param name="rdbFile"></param> private void CreateFullRatingTable(TextFile rdbFile) { // Build container DataTable DataTable ratingTable = new DataTable(); ratingTable.Columns.Add(new DataColumn("Stage", typeof(double))); ratingTable.Columns.Add(new DataColumn("Shift", typeof(double))); ratingTable.Columns.Add(new DataColumn("Flow", typeof(double))); // Save RDB file var tempFile = Path.GetTempFileName(); rdbFile.SaveAs(tempFile); rdbFile = new TextFile(tempFile); // Loop through each RDB file row to find the data int headerRow = rdbFile.IndexOf("INDEP SHIFT DEP STOR") + 2; for (int i = headerRow; i < rdbFile.Length; i++) { var row = rdbFile[i]; var newRow = ratingTable.NewRow(); var rowVals = rdbFile[i].Split('\t'); newRow["Stage"] = Convert.ToDouble(rowVals[0]); newRow["Shift"] = Convert.ToDouble(rowVals[1]); newRow["Flow"] = Convert.ToDouble(rowVals[2]); ratingTable.Rows.Add(newRow); } this.fullRatingTable = ratingTable; }
/// <summary> /// This method generates the HJ and Q tables /// </summary> /// <param name="station"></param> private void CreateShiftAndFlowTables(TextFile rdbFile) { var tempFile = Path.GetTempFileName(); rdbFile.SaveAs(tempFile); rdbFile = new TextFile(tempFile); var rdbFileString = rdbFile.FileContents; var rdbItems = rdbFileString.Split('\t', '\n', '\r', ' ').ToList();//[JR] these separate the rdb file into chunks, make sure that new line characters are specified in this line var skelPtIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i] == "*").ToList();//get skeletal points idx var breakptIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i].Contains("BREAKPOINT")).ToList();//get breakpoint idx var offsetIdx = Enumerable.Range(0, rdbItems.Count).Where(i => rdbItems[i].Contains("OFFSET")).ToList();//get offset idx // Generate HJ Table by defining stage-shift pairs within a C# DataTable var hjTable = CreateHJTable(rdbFileString); // Define Logarithmic coefficient pairs in a C# DataTable var coeffTable = new DataTable(); coeffTable.Columns.Add(new DataColumn("breakpoint", typeof(double))); coeffTable.Columns.Add(new DataColumn("offset", typeof(double))); if (offsetIdx.Count == 0) // no breakpoint and coefficient data { var coeffRow = coeffTable.NewRow(); coeffRow["breakpoint"] = -999999999.99; coeffRow["offset"] = 0.0; coeffTable.Rows.Add(coeffRow); } else if (offsetIdx.Count == 1) { var coeffRow = coeffTable.NewRow(); coeffRow["breakpoint"] = -999999999.99; coeffRow["offset"] = Convert.ToDouble(rdbItems[offsetIdx[0]].Split('=')[1].ToString().Replace("\"", "")); coeffTable.Rows.Add(coeffRow); } else { for (int i = 0; i < offsetIdx.Count; i++) { var coeffRow = coeffTable.NewRow(); if (i == 0) { coeffRow["breakpoint"] = -999999999.99; } else { coeffRow["breakpoint"] = Convert.ToDouble(rdbItems[breakptIdx[i - 1]].Split('=')[1].ToString().Replace("\"", "")); } coeffRow["offset"] = Convert.ToDouble(rdbItems[offsetIdx[i]].Split('=')[1].ToString().Replace("\"", "")); coeffTable.Rows.Add(coeffRow); } } // Generate Q DataTable var qTable = new DataTable(); qTable.Columns.Add(new DataColumn("Stage", typeof(double))); qTable.Columns.Add(new DataColumn("Flow", typeof(double))); qTable.Columns.Add(new DataColumn("A-Coeff", typeof(double))); qTable.Columns.Add(new DataColumn("B-Coeff", typeof(double))); foreach (var item in skelPtIdx) { var qRow = qTable.NewRow(); double ghVal = Convert.ToDouble(rdbItems[item - 3]); double shiftVal = Convert.ToDouble(rdbItems[item - 2]); double stageVal = ghVal + shiftVal; qRow["Stage"] = stageVal; qRow["Flow"] = Convert.ToDouble(rdbItems[item - 1]); DataRow[] coeffMatch = coeffTable.Select("[breakpoint] <= '" + stageVal + "'"); qRow["A-Coeff"] = coeffMatch[coeffMatch.Count() - 1][1]; qRow["B-Coeff"] = 0.0; qTable.Rows.Add(qRow); } if (qTable.Rows.Count < 1) { throw new Exception("No skeletal points found for station: " + this.stationName); } this.hjTable = hjTable; this.qTable = qTable; }
/********************************** * * Create a tree file that is usable by pisces * * input : riverware rdf file * output : comma seperated tree file. * * * // example portion of input file. * (this input file is using snapshots in riverware) * ----------------------------------- * object_type: SnapShotObj * object_name: Most Likely 2 * slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow * END_SLOT_PREAMBLE * units: cfs * * // example output. There is no nesting of tree levels for now. * RiverwareName,Description,RiverwareDataType,Level,Units * Riverware Results,,,0, * Yakima River at Parker PARW,Yakima River at Parker PARW,Gage Outflow,1,cfs * Yakima River at Grandview,Yakima River at Grandview,Gage Inflow,1,cfs * ... * **********************************/ public static void AddRiverWareFileToDatabase(string rdfFilename, PiscesFolder parent, TimeSeriesDatabase db) { Reclamation.Core.TextFile tf = new Reclamation.Core.TextFile(rdfFilename); #region notes /* * SnapShotStyle... * ------------------------------------- * 2001-9-29 24:00 * 2001-9-30 24:00 * object_type: SnapShotObj * object_name: Most Likely 2 ### scernario name * slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow # object_name slotName are combined. * END_SLOT_PREAMBLE * units: cfs * * Regular Style ... * --------------------------------------- * END_COLUMN * END_SLOT * object_type: StreamGage * object_name: Yakima 202_0 at Easton EASW * slot_name: Gage Outflow * END_SLOT_PREAMBLE * units: cfs * scale: 1 * */ #endregion int number_of_runs = LookupNumberOfRuns(tf); PiscesFolder folder = parent; if (number_of_runs == 1) { folder = db.AddFolder(parent, Path.GetFileNameWithoutExtension(rdfFilename)); } int sz = tf.Length; // object_type and object_name should occur on consecutive lines. int index = tf.IndexOfBoth("object_name:", "slot_name:", 0); var objectList = new List <string>(); //list to avoid duplicates in tree Performance p1 = new Performance(); Performance p2 = new Performance(); p2.Pause(); int counter = 0; db.SuspendTreeUpdates(); var sc = db.GetSeriesCatalog(); Dictionary <string, int> objTypeID = new Dictionary <string, int>(); Dictionary <string, int> objNameID = new Dictionary <string, int>(); while (index < sz && index > 0) { //slot_name: Andrews Gage 12447390 at RM 3_5_Gage Outflow string slot_name = tf[index + 1].Substring(11); //Andrews Gage 12447390 at RM 3_5_Gage Outflow string object_type = tf[index - 1].Substring(13); string object_name = tf[index].Substring(13); string units = tf[index + 3].Substring(6).Trim(); string tag = object_name + ":" + slot_name; if (!objectList.Contains(tag)) { int scenarioNumber = -1; if (number_of_runs > 1) { scenarioNumber = 1; } RiverWareSeries s; if (object_type == "SnapShotObj") { s = new RiverWareSeries(rdfFilename, "", slot_name, scenarioNumber, true, units); } else { s = new RiverWareSeries(rdfFilename, object_name, slot_name, scenarioNumber, false, units); } s.Units = units; s.ConnectionString = ConnectionStringUtility.MakeFileNameRelative(s.ConnectionString, db.DataSource); p2.Continue(); if (object_type.Contains("Reservoir")) { object_type = "Reservoir"; } else if (object_type.Contains("Reach")) { object_type = "Reach"; } else if (object_type.Contains("Diversion")) { object_type = "Diversion"; } else if (object_type.Contains("Canal")) { object_type = "Canal"; } int id = sc.NextID(); if (!sc.FolderExists(object_type, folder.ID)) { objTypeID.Add(object_type, id); sc.AddFolder(object_type, id, folder.ID); id++; } if (!sc.FolderExists(object_name, objTypeID[object_type])) { objNameID.Add(object_name, id); sc.AddFolder(object_name, id, objTypeID[object_type]); id++; } sc.AddSeriesCatalogRow(s, id, objNameID[object_name], ""); objectList.Add(tag); } index = tf.IndexOfBoth("object_name:", "slot_name:", index + 2); counter++; } p1.Report("total"); p2.Report("db.add()"); //398.7732813 seconds elapsed. total //384.6792607 seconds elapsed. db.add() // disable tree refresh (doubles perf) // 255.9736646 seconds elapsed. total // 241.7702669 seconds elapsed. db.add() // implemented member ExternalDataSource //34.8756696 seconds elapsed. total //20.3753912 seconds elapsed. db.add() var convention = Reclamation.TimeSeries.RiverWare.ImportRiverWare.ScenarioConvention.Default; if (number_of_runs > 1) // Multiple runs. { // show dialog to allow water year naming or traces var dlg = new Reclamation.TimeSeries.RiverWare.ImportRiverWare(); if (dlg.ShowDialog() == System.Windows.Forms.DialogResult.OK) { convention = dlg.NamingConvention; } // Add Scenarios. var tblScen = db.GetScenarios(); for (int i = 0; i < number_of_runs; i++) { string name = "Run" + i; if (convention == RiverWare.ImportRiverWare.ScenarioConvention.ByYear) { name = (dlg.FirstYear + i).ToString(); } //string scenarioPath = ConnectionStringUtility.MakeFileNameRelative("FileName=" + item, DB.Filename); tblScen.AddScenarioRow(name, true, "ScenarioNumber=" + (i + 1).ToString()); } db.Server.SaveTable(tblScen); } db.Server.SaveTable(sc); db.ResumeTreeUpdates(); db.RefreshFolder(parent); }
public void WaterYearTrace() { string path = Globals.TestDataPath; List<string> args = new List<string>(); //-UxlsFileName=V:\PN6200\Models\BoiseRiverWare\BoiseModelData.xls -UDebugLevel=1 -UWaterYear=1943 -UFirstWaterYear=1919 args.Add(path + "\\RiverWare\\snakeControl.txt"); args.Add("c:\\temp"); args.Add("1927-10-04");// start date for RiverWare args.Add("12:00"); args.Add("1928-07-01"); args.Add("12:00"); args.Add("1DAY"); args.Add("-UXlsFileName=" +path + "\\RiverWare\\SnakeTestData.xls"); args.Add("-STrace=4"); Reclamation.RiverwareDmi.Program.Main(args.ToArray()); string fn = @"c:\temp\Local Inflow.MinidokaToMilner_Milner.txt"; TextFile tf = new TextFile(fn); File.Delete(fn); int idx = tf.IndexOf("start_date"); Assert.AreEqual(-2906, Convert.ToDouble(tf[idx + 1]), .6); Assert.AreEqual(-2864, Convert.ToDouble(tf[idx + 2]), .6); }
/// <summary> /// Reads Raw RBMS Text file. Adds DateTime and Value Columns /// </summary> /// <param name="filename"></param> /// <returns></returns> private static DataTable ReadRBMSFile(string filename) { TextFile tf = new TextFile(filename); int fieldCount = 0; if (tf.FileData.Length > 0) { fieldCount = tf[0].Split(',').Length; if (fieldCount < 4 || fieldCount > 5) { throw new Exception("Error: " + filename + "is not a valid RBMS File. It should have 4 or 5 columns of data"); } } else { throw new Exception("Error: There was not data in the input file"); } tf = null; CsvFile raw = null; if (fieldCount == 5) { string[] dataTypes = new string[] { "String", "String", "String", "String", "String" }; string[] fieldNames = new string[] { "DH", "Riser", "Date", "Time", "Measured" }; //DH-438-RS , 1,01-apr-2000,00:00:54 , 870.290 //string pattern = @"\s*[A-Za-z\-0-9]{3,}\s*,\s*\d+,\d{1,2}-[A-Za-z]{3}-\d{4},\d{2}:\d{2}:\d{2}\s*,\s*[0-9\.\-\+Ee]+"; raw = new CsvFile(filename, dataTypes, fieldNames); } else { string[] dataTypes = new string[] { "String", "String", "String", "String" }; string[] fieldNames = new string[] { "DH", "Riser", "Date", "Measured" }; //DH-259-RS , 1,01-apr-1991 00:00:32 , 866.960 //string pattern = @"\s*[A-Za-z\-0-9]{3,}\s*,\s*\d+,\d{1,2}-[A-Za-z]{3}-\d{4}\s\d{2}:\d{2}(:\d{2})?\s*,\s*[0-9\.\-\+Ee]+"; raw = new CsvFile(filename, dataTypes, fieldNames); } return CreateDateValueTableForDatabase(raw); // format with date time together. //DH-259-RS , 1,01-apr-1991 00:00:32 , 866.960 // format with date time csv. //DH-438-RS , 1,01-apr-2000,00:00:54 , 870.290 //Console.WriteLine(db.Rows.Count); // return raw; }
public TextFileCredentials(string filename) { m_fileName = filename; tf = new TextFile(m_fileName); }
public static TimeSeriesDatabaseDataSet.RatingTableDataTable ParseRatingTableData(TextFile tf, string cbtt, string pcode) { /* <table border="1" summary="Expanded Rating Table"> <tr align="center"> <th width="80px"><b>ft</b></th> <th width="80px"><b>cfs</b></th> </tr> * * Table Edit Date: 18-SEP-2014 02:22 <tr align="right"><td>2440.01</td><td>-704.00</td></tr> <tr align="right"><td>2440.02</td><td>-704.00</td></tr> <tr align="right"><td>2440.03</td><td>-704.00</td></tr> <tr align="right"><td>2440.04</td><td>-704.00</td></tr> */ TimeSeriesDatabaseDataSet.RatingTableDataTable t = new TimeSeriesDatabaseDataSet.RatingTableDataTable(); int idx = tf.IndexOfBothRegex("<th.*</th>", "<th.*</th>"); if (idx >= 0) { t.XUnits = Regex.Match(tf[idx], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value; t.YUnits = Regex.Match(tf[idx + 1], @"<th.*<b>(?<x>\w*)</b>").Groups[1].Value; } //<br>Table Edit Date: 18-SEP-2014 02:22<br>Today's Date: 23-SEP-2014 06:54 var idxDate = tf.IndexOfRegex("Table Edit Date:.*<br>"); if (idxDate >= 0) { t.EditDate = Regex.Match(tf[idxDate], "Table Edit Date:(?<date>.*)<br>").Groups[1].Value; } Regex re = new Regex(@"<tr.*?>(?<x>[\d\.\-\+]{1,12})</td><td>(?<y>[\d\.\-\+]{1,12})</td>"); for (int i = idx + 2; i < tf.Length; i++) { var m = re.Match(tf[i]); if (m.Success) { double x, y; if (double.TryParse(m.Groups["x"].Value, out x) ) { if (double.TryParse(m.Groups["y"].Value, out y) && System.Math.Abs(998877 - y) > 0.01 ) { t.AddRatingTableRow(x, y); } } } } t.Name = "Hydromet " + cbtt + " " + pcode; idx = tf.IndexOfRegex("<b>Station .+<br />"); if (idx >= 0) t.Name = Regex.Match(tf[idx], "<b>Station (.+)<br />").Groups[1].Value; return t; }
public UsgsRDBFile(string filename) { tf = new TextFile(filename); ParseFile(); }
private void ReadSnowDepth(DateTime t1, DateTime t2) { /* -------------------------------------------------------------------------------- Change In Snow Water Snow Snow Water Change In Site Name Date Time Equivalent Depth Equivalent Snow Depth ------------------------------------------------------------------------------ ATLANTA SUMMIT 01/07 0000 16.1 58.0 01/08 0000 16.1 57.0 0.0 -1.0 01/09 0000 16.0 57.0 -0.1 0.0 01/10 0000 15.9 57.0 -0.1 0.0 01/11 0000 15.9 56.0 0.0 -1.0 01/12 0000 16.0 56.0 0.1 0.0 01/13 0000 16.8 61.0 0.8 5.0 01/14 0000 17.4 64.0 0.6 3.0 * -------------------------------------------------------------------------------- Change In Snow Water Snow Snow Water Change In Site Name Date Time Equivalent Depth Equivalent Snow Depth ------------------------------------------------------------------------------ JACKSON PEAK 01/07 0000 13.5 50.0 01/08 0000 13.5 48.0 0.0 -2.0 01/09 0000 13.4 49.0 -0.1 1.0 01/10 0000 13.4 48.0 0.0 -1.0 01/11 0000 13.4 48.0 0.0 0.0 01/12 0000 13.5 -99.9 0.1 -99.9 01/13 0000 14.5 -99.9 1.0 -99.9 01/14 0000 15.4 62.0 0.9 -99.9 ------------------------------------------------------------------------------ ------------------------------------------------------------------------------ */ var data = Web.GetPage(urlDepth + state + "/" + siteName + ".txt", true); TextFile tf = new TextFile(data); var idx = tf.IndexOf("Site Name Date Time Equivalent Depth"); if (idx > 0) { string pattern = "^.{23}(?<month>\\d{2})/(?<day>\\d{2})\\s{1}0000\\s*(?<se>[0-9\\.+-]+)\\s*(?<sd>[0-9\\.+-]+)"; for (int i = idx+2; i < idx+10; i++) { Regex re = new Regex(pattern); var m = re.Match(tf[i]); if (m.Success) { int month = int.Parse(m.Groups["month"].Value); int day = int.Parse(m.Groups["day"].Value); double sd = double.Parse(m.Groups["sd"].Value); DateTime t = new DateTime(DateTime.Now.Year, month, day); if (t >= t1 && t <= t2) { if (System.Math.Abs(sd - 99.9) < 0.1) AddMissing(t); Add(t, sd); } } } } }