private static void SaveToFile(System.Data.DataRow row, Series s, string dir) { var cbtt = row["cbtt"].ToString(); var pcode = row["pcode"].ToString(); var d = DateTime.Now.ToString("yyyyMMddHHmm"); var interval = row["interval"].ToString(); if (interval.ToLower() == "instant") { s.TimeInterval = TimeInterval.Irregular; var fn = Path.Combine(dir, "instant_ipco_" + cbtt + "_" + pcode + d + ".txt"); Console.WriteLine(fn); HydrometInstantSeries.WriteToHydrometFile(s, cbtt, pcode, "idahopower", fn); } else if (interval.ToLower() == "daily") { s.TimeInterval = TimeInterval.Daily; var fn = Path.Combine(dir, "daily_ipco_" + cbtt + "_" + pcode + d + ".txt"); Console.WriteLine(fn); HydrometDailySeries.WriteToArcImportFile(s, cbtt, pcode, fn); } }
private void WriteListToFile(SeriesList list, string name, TimeInterval interval, string route) { var tmpFileName = FileUtility.GetTempFileName(".txt"); File.Delete(tmpFileName); Logger.WriteLine("writing " + list.Count + " series "); Logger.WriteLine("temp file:" + tmpFileName); if (interval == TimeInterval.Daily) { SeriesList exportList = new SeriesList(); foreach (var s in list) { exportList.Add(s); } HydrometDailySeries.WriteToArcImportFile(exportList, tmpFileName); } if (interval == TimeInterval.Irregular) { foreach (var s in list) { HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, Environment.UserName, tmpFileName, true); } } if (File.Exists(tmpFileName)) { Logger.WriteLine("Moving: " + tmpFileName); string fn = interval == TimeInterval.Daily ? "daily" : "instant"; var fileName = GetOutgoingFileName(fn, name, "all", route); Logger.WriteLine("To: " + fileName); File.Move(tmpFileName, fileName); } }
/// <summary> /// Routes a list of Series as a group /// hydromet cbtt is copied from list[i].SiteName /// hydromet pcode is copied from list[i].Parameter /// </summary> /// <param name="list"></param> /// <param name="route"></param> /// <param name="name">identity used as part of filename </param> public static void RouteInstant(SeriesList list, string name, RouteOptions route = RouteOptions.Both) { if (route == RouteOptions.None) { return; } if (route == RouteOptions.Both || route == RouteOptions.Outgoing) { var tmpFileName = FileUtility.GetTempFileName(".txt"); File.Delete(tmpFileName); Console.WriteLine("temp file:" + tmpFileName); foreach (var s in list) { HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, WindowsUtility.GetShortUserName(), tmpFileName, true); } Console.WriteLine("Moving: " + tmpFileName); var fileName = GetOutgoingFileName("instant", name, "all"); Console.WriteLine("To: " + fileName); File.Move(tmpFileName, fileName); } else { throw new NotImplementedException("incoming not supported."); } }
/// <summary> /// Lookup hydromet Series. /// name is interval_cbtt_pcode /// </summary> /// <param name="name"></param> /// <returns></returns> public override ParserResult Lookup(string name, TimeInterval defaultInterval) { var interval = defaultInterval; TimeSeriesName tn = new TimeSeriesName(name); if( tn.HasInterval) { interval = tn.GetTimeInterval(); } if (tn.Valid) { Logger.WriteLine("Hydromet Lookup " + tn.siteid + "," + tn.pcode); var s = new Series(); if (interval == TimeInterval.Monthly) { s = new HydrometMonthlySeries(tn.siteid, tn.pcode,svr); } else if (interval == TimeInterval.Irregular) { s = new HydrometInstantSeries(tn.siteid, tn.pcode,svr); } else if (interval == TimeInterval.Daily) { s = new HydrometDailySeries(tn.siteid, tn.pcode,svr); } return new ParserResult(s); } else { return base.Lookup(name,interval); } }
//http://imapx.codeplex.com/documentation static void Main(string[] args) { // ApplicationTrustPolicy.TrustAll(); string credientials = ConfigurationManager.AppSettings["credentials"]; string imap_server = ConfigurationManager.AppSettings["imap_server"]; string logfilename = ConfigurationManager.AppSettings["log"]; string output_dir = ConfigurationManager.AppSettings["output_dir"]; string email_folder = ConfigurationManager.AppSettings["email_folder"]; var client = new ImapClient(imap_server, true, false); if (logfilename != null) { Logger.EnableLogger(); Logger.WriteLine("logging to '" + logfilename + "'"); Debug.Listeners.Add(new TextWriterTraceListener(logfilename)); Debug.AutoFlush = true; client.IsDebug = true; } var files = Directory.GetFiles(credientials, "*.*"); var user = Path.GetFileName(files[0]); var pass = File.ReadAllLines(files[0])[0]; Logger.WriteLine("about to connect:"); if (client.Connect()) { if (client.Login(user, pass)) { // login successful Console.WriteLine("ok"); Logger.WriteLine("listing labels..."); foreach (var item in client.Folders) { Logger.WriteLine(item.Name); } var folder = client.Folders[email_folder]; folder.Messages.Download("UNSEEN", MessageFetchMode.Basic | MessageFetchMode.GMailExtendedData, 72); var s = GetSerieData(folder); var cbtt = "mpci"; var pcode = "qp2"; string fn = "instant_" + cbtt + "_" + pcode + "_" + DateTime.Now.ToString("MMMdyyyyHHmmssfff") + ".txt"; fn = Path.Combine(output_dir, fn); HydrometInstantSeries.WriteToHydrometFile(s, cbtt, pcode, "hydromet", fn); } } else { Logger.WriteLine("connection not successful"); } }
public Series FetchInstantFromAPI(string site, string pcode, DateTime start, DateTime end) { // TODO: Add caching Series s = new Series("", TimeInterval.Irregular); s.Table = HydrometInstantSeries.Read(site, pcode, start, end, HydrometHost.PNLinux).Table; s.Table.TableName = "instant_" + site + "_" + pcode; s.Name = s.Table.TableName; return(s); }
// Populates Pisces DB with instant 15-minute data private static Series GetInstantSeries(string CBTT, string PCODE, DateTime t1, DateTime t2, TimeSeriesDatabase pDB, PiscesFolder rFldr) { Console.Write(CBTT + "_" + PCODE + ", "); Series rval = new HydrometInstantSeries(CBTT, PCODE); rval.Read(t1, t2); rval.Name = CBTT + "_" + PCODE + "15min"; pDB.AddSeries(rval, rFldr); return(rval); }
private static void ProcessFile(string filename, string outputFilename, CsvFile csv) { for (int i = 0; i < csv.Rows.Count; i++) { var r = csv.Rows[i]; var s = SimpleShef.ReadSimpleShefA(filename, r["shefloc"].ToString(), r["shefcode"].ToString()); Console.WriteLine(r["cbtt"].ToString() + "/" + r["pcode"].ToString() + " " + s.Count + " records"); HydrometInstantSeries.WriteToHydrometFile(s, r["cbtt"].ToString(), r["pcode"].ToString(), "klamath", outputFilename, true); } }
private static void ProcessFile(string filename, string outputFilename, CsvFile csv) { for (int i = 0; i < csv.Rows.Count; i++) { var r = csv.Rows[i]; var s = SimpleShef.ReadSimpleShefA(filename, r["shefloc"].ToString(), r["shefcode"].ToString()); HydrometInstantSeries.WriteToHydrometFile(s, r["cbtt"].ToString(), r["pcode"].ToString(), WindowsUtility.GetUserName(), outputFilename, true); } }
public double GetValue(string cbtt, string pcode, DateTime t) { var s = new HydrometInstantSeries(cbtt, pcode); //DateTime th = new DateTime(t.Year, t.Month, t.Day, hour, 0, 0); s.Read(t, t); if (s.Count > 0 && !s[0].IsMissing) { return(s[0].Value); } return(Point.MissingValueFlag); }
public static void Import(Series s, string siteID, string parameter, string fileExtension = "txt") { if (s.TimeInterval == TimeInterval.Daily) { var fn = TimeSeriesTransfer.GetIncommingFileName("daily", siteID, parameter, fileExtension); HydrometDailySeries.WriteToArcImportFile(s, siteID, parameter, fn); } if (s.TimeInterval == TimeInterval.Irregular || s.TimeInterval == TimeInterval.Hourly) { var fn = TimeSeriesTransfer.GetIncommingFileName("instant", siteID, parameter, fileExtension); HydrometInstantSeries.WriteToHydrometFile(s, siteID, parameter, Environment.UserName, fn); } }
static Series GetInstantSeries(string cbtt, string pcode, DateTime t, int hours_back) { Series s = new HydrometInstantSeries(cbtt, pcode, Utility.GetHydrometServer()); var t1 = t.AddHours(-hours_back); s.Read(t1, t); // Hydromet web query doesn't have hour resolution. we must clean that ourselves. var s2 = Reclamation.TimeSeries.Math.Subset(s, t1, t); // keep data in HydrometInstantSeries (becuse it understands it's flags) s.Clear(); s.Add(s2); return(s); }
/// <summary> /// Imports all required Series data to satisfy required equation inputs. /// </summary> private void ReadAllData(DateTime t1, DateTime t2, out Series dewTempDataAll, out Series windDataAll, out Series avgTempDataAll, out Series R_s1All) { string cbtt = this.cbtt; dewTempDataAll = new HydrometInstantSeries(cbtt, "TP"); //Dew Point Temp (DegF) dewTempDataAll.Read(t1, t2); windDataAll = new HydrometInstantSeries(cbtt, "WS"); //Wind Speed (mph) windDataAll.Read(t1, t2); avgTempDataAll = new HydrometInstantSeries(cbtt, "OB"); //Average Temp (DegF) avgTempDataAll.Read(t1, t2); R_s1All = new HydrometInstantSeries(cbtt, "SI"); //Solar Radiation (MJ/m2/hr) R_s1All.Read(t1, t2); }
/// <summary> /// .A LWOI1 080313 M DH0530/QRIRG 72.07 /// .A LWOI1 080313 M DH0545/QRIRG 69.50 /// .A LWOI1 080313 M DH0600/QRIRG 70.78 /// .A LWOI1 080313 M DH0615/QRIRG 69.50 /// </summary> private static void WriteHourly(DateTime t1, DateTime t2, StreamWriter sw, string cbtt, string pcode, string shefName, string shefPcode, string timeZone) { //HydrometInstantSeries.UseCaching = false; Series s = HydrometInstantSeries.Read(cbtt, pcode, t1, t2, HydrometHost.PNLinux); s.RemoveMissing(); for (int j = 0; j < s.Count; j++) { DateTime t = s[j].DateTime; sw.WriteLine(".A " + shefName + " " + t.ToString("yyMMdd") + " " + timeZone + " " + "DH" + t.ToString("HHmm") + "/" + shefPcode + " " + s[j].Value.ToString("F2")); } System.Threading.Thread.Sleep(100); }
static void Main(string[] args) { string dbFileName = "test.pdb"; if (File.Exists(dbFileName)) { File.Delete(dbFileName); } SQLiteServer sqlite = new SQLiteServer(dbFileName); TimeSeriesDatabase db = new TimeSeriesDatabase(sqlite); // read seriescatalog.csv -- from the production database. This is a template without the data DataTable csv = new CsvFile("seriescatalog.csv"); //get a list of site that have instant (irregular) data string[] sites = GetSitesWithPrecip(csv); // for each site with precipitation // ------------- for (int i = 0; i < sites.Length; i++) { // add instant precip (pc -- raw in the bucket values) var folder = db.GetOrCreateFolder(sites[i]); Series s = new Series("", TimeInterval.Irregular); s.Table = HydrometInstantSeries.Read(sites[i], "pc", DateTime.Parse("2010-10-1"), DateTime.Now.Date, HydrometHost.PNLinux).Table; s.Table.TableName = "instant_" + sites[i] + "_pc"; s.Name = s.Table.TableName; db.AddSeries(s, folder); if (i > 3) // make testing faster { break; } // add daily precip (pc -- midnight value of the bucket) // add daily incremental ( pp -- daily sum of precipitation) // add daily cumulative (pu -- water year cummulative) } }
public static void RouteInstant(Series s, string cbtt, string pcode, RouteOptions route = RouteOptions.Both) { if (s.Count == 0) { return; } string fileName = ""; if (route == RouteOptions.Both || route == RouteOptions.Outgoing) { fileName = GetOutgoingFileName("instant", cbtt, pcode); HydrometInstantSeries.WriteToHydrometFile(s, cbtt, pcode, WindowsUtility.GetShortUserName(), fileName); } if (route == RouteOptions.Both || route == RouteOptions.Incoming) { fileName = GetIncommingFileName("instant", cbtt, pcode); HydrometInstantSeries.WriteToHydrometFile(s, cbtt, pcode, WindowsUtility.GetShortUserName(), fileName); } }
/// <summary> /// Lookup hydromet Series. /// name is interval_cbtt_pcode /// </summary> /// <param name="name"></param> /// <returns></returns> public override ParserResult Lookup(string name, TimeInterval defaultInterval) { var interval = defaultInterval; TimeSeriesName tn = new TimeSeriesName(name); if (tn.HasInterval) { interval = tn.GetTimeInterval(); } if (tn.Valid) { Logger.WriteLine("Hydromet Lookup " + tn.siteid + "," + tn.pcode); var s = new Series(); if (interval == TimeInterval.Monthly) { s = new HydrometMonthlySeries(tn.siteid, tn.pcode, svr); } else if (interval == TimeInterval.Irregular) { s = new HydrometInstantSeries(tn.siteid, tn.pcode, svr); } else if (interval == TimeInterval.Daily) { s = new HydrometDailySeries(tn.siteid, tn.pcode, svr); } return(new ParserResult(s)); } else { return(base.Lookup(name, interval)); } }
public void GraphData(bool showFlows = false) { linkLabelReport.Visible = false; if (comboBoxSite.Text.ToString() == "" || this.comboBoxSite.SelectedItem.ToString().IndexOf("--") != -1) { return; } try { optionalPercents = ParseOptionalPercentages(); Series alternateRequiredContent = new Series(); Series alternateActualContent = new Series(); Series actualContent = new Series(); Series requiredContent = new Series(); SeriesList targets = new SeriesList(); SeriesList hmList = new SeriesList(); SeriesList hmList2 = new SeriesList(); SeriesList ruleCurves = new SeriesList(); var cache = new HydrometDataCache(); var cbttList = new List <string>(); var cbttListAlternate = new List <string>(); var labelDates = new List <DateTime>(); bool showRuleCurve = true; Reclamation.TimeSeries.Hydromet.HydrometHost svr; var hydSvr = UserPreference.Lookup("HydrometServer"); if (hydSvr.ToLower() == "greatplains") { svr = HydrometHost.GreatPlains; } else { svr = HydrometHost.PNLinux; } Cursor = Cursors.WaitCursor; Application.DoEvents(); FloodControlPoint pt = new FloodControlPoint(this.comboBoxSite.Text.ToString()); if (compilePublic) { pt.StationFC = "NA"; pt.StationQU = "NA"; pt.RequiredLegend = "Full Storage"; showRuleCurve = false; } if (pt.DailyStationQU == "NA") { return; } if (showFlows) { this.pcodeInitial.Text = (pt.ResOpsInflow + "," + pt.ResOpsOutflow).ToLower(); hydrometChart1.SetRightLabels("Flows (cfs)"); } checkBoxDashed.Visible = pt.StationFC.ToLower() == "heii"; residForecast = new ResidualForecast(pt, checkBoxDashed.Checked); residForecastCompare = new ResidualForecast(pt, checkBoxDashed.Checked); DateRange requiredRange = GetRequiredRange(); DateRange curveRange = GetRuleCurveRange(); DateRange alternateRange = GetComparisonRange(); //setup cache for required cbttList.AddRange(residForecast.DailyCbttPcodeList()); if (this.pcodeInitial.Text.Length >= 1 && this.textBoxWaterYear.Text.Length >= 1) { cbttList.AddRange(OptionalCbttList(this.pcodeInitial.Text)); } cache.Add(cbttList.ToArray(), requiredRange.DateTime1.AddDays(-1), requiredRange.DateTime2.AddDays(1), svr, TimeInterval.Daily); cache.Add(residForecast.MonthlyCbttPcodeList(), requiredRange.DateTime1, requiredRange.DateTime2, svr, TimeInterval.Monthly); // setup cache for alternate range if (alternateRange.IsValid) { cbttListAlternate.AddRange(residForecast.DailyCbttPcodeList()); if (this.pcodeComparison.Text.Length >= 1) { cbttListAlternate.AddRange(OptionalCbttList(this.pcodeComparison.Text)); } cache.Add(cbttListAlternate.ToArray(), alternateRange.DateTime1.AddDays(-1), alternateRange.DateTime2.AddDays(1), svr, TimeInterval.Daily); cache.Add(residForecast.MonthlyCbttPcodeList(), alternateRange.DateTime1.AddDays(-1), alternateRange.DateTime2.AddDays(1), svr, TimeInterval.Monthly); } //add cache HydrometDailySeries.Cache = cache; HydrometMonthlySeries.Cache = cache; //compute residual forecast residForecast.Compute(requiredRange.DateTime1, requiredRange.DateTime2, svr); requiredContent = -residForecast.SpaceRequired + pt.TotalUpstreamActiveSpace; actualContent = residForecast.TotalContent; requiredContent.Name = this.textBoxWaterYear.Text; actualContent.Name = this.textBoxWaterYear.Text + " Actual"; if (this.pcodeInitial.Text.Length >= 1) { hmList = ReadHydrometOptionalData(Convert.ToInt32(this.textBoxWaterYear.Text), this.pcodeInitial.Text, requiredRange); } //compute comparison year residual forecast if (alternateRange.IsValid) { residForecastCompare.Compute(alternateRange.DateTime1, alternateRange.DateTime2, svr); alternateRequiredContent = -residForecastCompare.SpaceRequired + pt.TotalUpstreamActiveSpace; alternateRequiredContent.Name = this.textBoxAlternateWaterYear.Text; alternateActualContent = residForecastCompare.TotalContent; alternateActualContent.Name = this.textBoxAlternateWaterYear.Text + " Actual"; if (this.pcodeComparison.Text.Length >= 1) { hmList2 = ReadHydrometOptionalData(Convert.ToInt32(this.textBoxAlternateWaterYear.Text), this.pcodeComparison.Text, alternateRange); hmList.AddRange(hmList2); } } if (showGreenLines.Checked == true && !compilePublic) // display flood rule curves for various forecast levels { showRuleCurve = true; // Green lines if (residForecast.RuleCurve.FillType == FillType.Fixed) { ruleCurves = residForecast.RuleCurve.CalculateFixedRuleCurve(curveRange.DateTime1, curveRange.DateTime2, pt.TotalUpstreamActiveSpace); labelDates.Add(curveRange.DateTime1); } else { ruleCurves = residForecast.RuleCurve.CalculateVariableRuleCurves(curveRange.DateTime1, curveRange.DateTime2, pt.TotalUpstreamActiveSpace, pt.PercentSpace / 100.0); labelDates.AddRange(FcPlotDataSet.GetVariableForecastLabelDates(residForecast.RuleCurve.CurveName)); } } else { showRuleCurve = false; } hydrometChart1.SetLabels(pt.Name, "Storage Content (acre-feet)"); bool dashedLines = checkBoxDashed.Checked && pt.StationFC.ToLower() == "heii"; // DEFAULT - Plots daily storage content data if (!checkBoxShowInstantAF.Checked && !checkBoxShowElevation.Checked) { hydrometChart1.Fcplot(residForecast.TotalContent, requiredContent, alternateRequiredContent, alternateActualContent, ruleCurves, labelDates.ToArray(), pt.RequiredLegend, hmList, showRuleCurve, dashedLines); //compute the targets if (pt.FillType == FillType.Variable && (showTarget.Checked == true || checkBoxOverrideFcast.Checked == true)) { if (Convert.ToInt32(this.textBoxWaterYear.Text) == DateTime.Now.WaterYear()) { actualContent.RemoveMissing(); var startPt = actualContent[actualContent.Count - 1]; targets = FloodOperation.ComputeTargets(pt, Convert.ToInt32(this.textBoxWaterYear.Text), startPt, optionalPercents, checkBoxDashed.Checked, this.checkBoxOverrideFcast.Checked, this.textBoxOverrideFcast.Text); var aColors = new Color[] { Color.Black, Color.Maroon, Color.Indigo, Color.DarkSlateGray, Color.SaddleBrown }; for (int i = 0; i < targets.Count; i++) { var s = targets[i]; var c = Color.Black; if (i < aColors.Length) { c = aColors[i]; } hydrometChart1.CreateTarget(c, s.Name, s, "left"); // Add target to datatable residForecast.AddColumnToReportTable(s); } } } } else { string leftPcode = "af";; if (this.checkBoxShowElevation.Checked) { leftPcode = "fb"; hydrometChart1.SetLabels(pt.Name, "Water Level Elevation (feet)"); if (alternateRange.IsValid) { var sAlternate = new Series(); if (checkBoxShowInstantAF.Checked) { sAlternate = new HydrometInstantSeries(pt.UpstreamReservoirs[0], leftPcode, svr); } else { sAlternate = new HydrometDailySeries(pt.UpstreamReservoirs[0], leftPcode, svr); } sAlternate.Read(alternateRange.DateTime1, alternateRange.DateTime2); sAlternate.Name = this.textBoxAlternateWaterYear.Text + " Actual"; sAlternate.Units = residForecast.TotalContent.Units; alternateActualContent = sAlternate; } } if (pt.UpstreamReservoirs.Length == 1) { var s = new Series(); if (checkBoxShowInstantAF.Checked) { var sInstant = new HydrometInstantSeries(pt.UpstreamReservoirs[0], leftPcode, svr); s = sInstant; } else { var sDaily = new HydrometDailySeries(pt.UpstreamReservoirs[0], leftPcode, svr); s = sDaily; } s.Read(residForecast.TotalContent.MinDateTime, residForecast.TotalContent.MaxDateTime.AddDays(1)); s.Name = residForecast.TotalContent.Name; s.Units = residForecast.TotalContent.Units; hydrometChart1.Fcplot(s, requiredContent, alternateRequiredContent, alternateActualContent, ruleCurves, labelDates.ToArray(), pt.RequiredLegend, hmList, showRuleCurve, dashedLines, checkBoxShowElevation.Checked); } } linkLabelReport.Visible = true; labelFlagLegend.Text = pt.FlagLegend; dataGridView1.DataSource = residForecast.ReportTable; } catch (Exception ex) { MessageBox.Show(ex.Message); } finally { Cursor = Cursors.Default; } Cursor = Cursors.Default; }
// Populates Pisces DB with instant 15-minute data private static Series GetInstantSeries(string CBTT, string PCODE, DateTime t1, DateTime t2, TimeSeriesDatabase pDB, PiscesFolder rFldr) { Console.Write(CBTT + "_" + PCODE + ", "); Series rval = new HydrometInstantSeries(CBTT, PCODE); rval.Read(t1, t2); rval.Name = CBTT + "_" + PCODE + "15min"; pDB.AddSeries(rval, rFldr); return rval; }
private void ProcessFile(RouteOptions routing, FileSystemInfo fi) { var fn = fi.FullName; string dir = System.IO.Path.GetDirectoryName(fn); if (fi.CreationTime.AddSeconds(2) > DateTime.Now) { Console.WriteLine(" skipping file newer than 2 seconds ago " + fn + " " + fi.CreationTime); return; } string importTag = "import"; // used to make friendly export filename try { TextFile tf = new TextFile(fi.FullName); SeriesList sl = new SeriesList(); if (HydrometInstantSeries.IsValidDMS3(tf)) { importTag = "decodes"; sl = Reclamation.TimeSeries.Hydromet.HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf); } else if (LoggerNetFile.IsValidFile(tf)) { LoggerNetFile lf = new LoggerNetFile(tf); if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0) { importTag = lf.SiteName; sl = lf.ToSeries(validPcodes); } } else if (DecodesRawFile.IsValidFile(tf)) { DecodesRawFile df = new DecodesRawFile(tf); importTag = "raw"; sl = df.ToSeries(); } else { Console.WriteLine("skipped Unknown File Format: " + fn); return; } m_importer = new TimeSeriesImporter(m_db, routing); Console.WriteLine("Found " + sl.Count + " series in " + fn); foreach (var item in sl) { Logger.WriteLine(item.Table.TableName); } if (sl.Count > 0) { m_importer.Import(sl, m_computeDependencies, m_computeDailyOnMidnight, importTag); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fn), "attic", fn); } } catch (Exception ex) { Logger.WriteLine("Error:" + ex.Message); Console.WriteLine("Error: skipping file, will move to error subdirectory " + fn); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fn), "error", fn); } }
static void Main(string[] argList) { //Logger.EnableLogger(); //These could be populated from a config file string[] HydrometParameterCodes = new string[] { "SI", "OBM", "TU", "WS", "WD", "WG", "BP", "PC", "TP" }; string[] NiceNetParameterNames = new string[] { "SolarRadKw/m2", "AveAirTempF", "AveRelHum%", "AveWindSpeedmph", "VectorWindDirdeg", "MaxWindGustmph", "BaroPressmb", "TotalPrecipin", "DewPointTemp" }; double[] NiceNetParameterUpperLimits = new double[] { 120, 120, 101, 100, 360, 100, 32, 51, 100 }; double[] NiceNetParameterLowerLimits = new double[] { 0, -50, 0, 0, 0, 0, 27, 0, -35 }; //SI comes in as Kw/m2 and is converted to ly/hr string[] NiceNetParameterUnits = new string[] { "ly/hr", "deg F", "%", "mph", "deg", "mph", "mb", "in", "deg F" }; var ParameterSet = new Dictionary <string, Parameter>(); for (int i = 0; i < HydrometParameterCodes.Length; i++) { var tempParameter = new Parameter(); tempParameter.UpperLimitValue = NiceNetParameterUpperLimits[i]; tempParameter.LowerLimitValue = NiceNetParameterLowerLimits[i]; tempParameter.UpperLimitFlag = "+"; tempParameter.LowerLimitFlag = "-"; tempParameter.Name = NiceNetParameterNames[i]; tempParameter.Units = NiceNetParameterUnits[i]; tempParameter.Code = HydrometParameterCodes[i]; ParameterSet.Add(HydrometParameterCodes[i], tempParameter); } var titleSeparator = new string[] { " ", "." }; if (argList.Length == 0) { Usage(); return; } Arguments args = new Arguments(argList); if (!args.Contains("config")) { Console.WriteLine("Error: --config=filename.csv is required"); Usage(); return; } if (!args.Contains("output")) { Console.WriteLine("Error: --output=filename.txt is required"); Usage(); return; } int hoursBack = 4; if (args.Contains("back")) { hoursBack = int.Parse(args["back"]); } //FileUtility.CleanTempPath(); //Read config file and filter on cbtt if a particular one is specified. DataTable csv = new CsvFile(args["config"], CsvFile.FieldTypes.AllText); if (args.Contains("cbtt")) { Console.WriteLine("Filtering for cbtt = '" + args["cbtt"] + "'"); csv = DataTableUtility.Select(csv, "cbtt='" + args["cbtt"] + "'", ""); } var rows = csv.Select(); //Selects all rows of config file, excluding the header row Console.WriteLine("Processing data for " + rows.Length + " site(s)"); Dictionary <string, string> bumParameters = new Dictionary <string, string>(); List <String> bumSites = new List <string>(); // Begin loop to read in data for each site. for (int i = 0; i < rows.Length; i++) { //Get site/cbtt from config file var site = rows[i]["dri_id"].ToString(); var cbtt = rows[i]["cbtt"].ToString(); var obm = new Series(); var tu = new Series(); Console.WriteLine("Processing site " + site + "/" + cbtt); // example http://www.wrcc.dri.edu/cgi-bin/nclvLIST.pl string url = "http://www.wrcc.dri.edu/cgi-bin/" + site.ToLower() + "LIST.pl"; //Catch missing web pages try { string[] dataPage = Web.GetPage(url); //Work through unwanted header rows of NiceNet data file. int j = 0; while (dataPage[j].Contains("<")) { j++; } //Get column header names //Revise this so that column order can change with out breaking the code. Might need to assume fixed width. var columnNames = dataPage[j].Split(titleSeparator, StringSplitOptions.RemoveEmptyEntries).ToList(); j++; while (!dataPage[j].Contains("--")) { var tempColumnNames = dataPage[j].Split(titleSeparator, StringSplitOptions.RemoveEmptyEntries).ToList(); if (tempColumnNames.Contains("Year")) { tempColumnNames.Insert(2, ""); } for (int n = 0; n < tempColumnNames.Count; n++) { columnNames[n] = columnNames[n] + tempColumnNames[n]; } j++; } //Get year from header var lastHeaderRow = j - 1; var startDataRow = j + 1; var dataHeaderRow = dataPage[lastHeaderRow].Split(titleSeparator, StringSplitOptions.RemoveEmptyEntries).ToList(); var year = Int32.Parse(dataHeaderRow[0]); //Process data for each parameter of current site and save to hydrometfile //First two parameters are day and time foreach (var p in ParameterSet) { //Assign PCode and look for missing/unexpected parameters in the //data file. //Skip dew point (TP). It is not in the input file. It is calculated later. if (p.Key == "TP") { continue; } try { var pCode = p.Key; var dataIndex = columnNames.IndexOf(p.Value.Name); var unitType = p.Value.Units; var s = ProcessDataSeries(cbtt, year, unitType, pCode, dataIndex, dataPage, startDataRow); s = ApplyFlags(s, p.Value); if (pCode == "OBM") { obm = s; } if (pCode == "TU") { tu = s; } //Only load the last hoursBack hours of data s.Trim(DateTime.Now.AddHours(-hoursBack), DateTime.Now.AddHours(2)); HydrometInstantSeries.WriteToHydrometFile(s, cbtt, pCode, "nicenet", args["output"], true); } catch { Console.WriteLine("Parameter \"" + p + "\" is not recognized."); } } var tp = DewPointCalculation(obm, tu, cbtt); if (tp.Count > 0) { tp = ApplyFlags(tp, ParameterSet["TP"]); HydrometInstantSeries.WriteToHydrometFile(tp, cbtt, "TP", "nicenet", args["output"], true); } else { Console.WriteLine("Dew point temperature could not be calculated."); } } catch (Exception e) { Console.WriteLine(e.Message); bumSites.Add(site); } } if (bumSites.Count >= 1) { Console.WriteLine("The following sites were not found:"); Console.WriteLine(String.Join("\n", bumSites.ToArray())); } }
private void buttonRefresh_Click(object sender, EventArgs e) { try { timeSeriesGraph1.AnnotationOnMouseMove = checkBoxAnnotate.Checked; Cursor = Cursors.WaitCursor; Application.DoEvents(); string pcodeOrig = DeterminePcode(); timeSeriesGraph1.Clear(); string cbttOrig = comboBoxCbtt.Text.Trim(); string cbtt = cbttOrig, pcode = pcodeOrig; var seriesList = new List <string>(); if ((cbttOrig.Trim() == "" || pcodeOrig.Trim() == "") && textBoxMultiple.Text == "") { return; } else { if (!checkBoxUseList.Checked) { UserPreference.Save("Snowgg->cbtt", cbttOrig); UserPreference.Save("Snowgg->pcode", comboBoxPcode.Text.Trim()); seriesList.Add(cbttOrig + "_" + pcodeOrig); } else { var seriesItems = textBoxMultiple.Text.Split(','); foreach (string item in seriesItems) { if (item.Trim().Split(' ').Length == 2) { seriesList.Add(item.Trim().Split(' ')[0] + "_" + item.Trim().Split(' ')[1]); } } } } int[] waterYears = this.yearSelector1.SelectedYears; SeriesList finalSeriesCollection = new SeriesList(); foreach (string series in seriesList) { cbtt = series.Split('_')[0]; comboBoxCbtt.Text = cbtt; pcode = series.Split('_')[1]; comboBoxPcode.Text = pcode; var server = HydrometInfoUtility.HydrometServerFromPreferences(); var range = monthRangePicker1.MonthDayRange; Series s; if (this.checkBoxUseInstant.Checked) { s = new HydrometInstantSeries(cbtt, pcode, server); } else { s = new HydrometDailySeries(cbtt, pcode, server); } var sl = new SeriesList(); sl.Add(s); // get wy data var wyList = new SeriesList(); if (cySelected) { wyList = PiscesAnalysis.WaterYears(sl, waterYears, false, 1, true); } else { wyList = PiscesAnalysis.WaterYears(sl, waterYears, false, 10, true); } foreach (Series item in wyList) { item.Name = cbtt + " " + pcode; // remove missing data points var missingItems = item.Table.Select("value = 998877"); foreach (var row in missingItems) { item.RemoveAt(item.IndexOf(Convert.ToDateTime(row.ItemArray[0]))); } } // apply deltas and add stats if toggled wyList = ApplyDeltas(wyList, waterYears); AddStatistics(wyList); if (checkBoxGP.Checked) { GPAverage(cbtt, server, range, wyList); } var mp = ReadMpollData(pcode, cbtt); mp.RemoveMissing(); if (mp.Count > 0) { wyList.Add(mp); } // remove months outside selected range var list = FilterBySelectedRange(range, wyList); finalSeriesCollection.Add(list); } // Set series line colors var uniqueSeriesNames = new List <string>(); var uniqueSeriesColors = new List <string>(); int colorCounter = 0; foreach (var item in finalSeriesCollection) { // set line color by year which is identified in the legendtext field if (!uniqueSeriesNames.Contains(item.Appearance.LegendText) && !item.Appearance.LegendText.Contains("%") && !item.Appearance.LegendText.Contains("avg") && !item.Appearance.LegendText.Contains("max") && !item.Appearance.LegendText.Contains("min")) { uniqueSeriesNames.Add(item.Appearance.LegendText);//.Name); uniqueSeriesColors.Add(snowGgColors[colorCounter]); colorCounter = (colorCounter + 1) % snowGgColors.Count; } } foreach (var item in finalSeriesCollection) { try { int colIdx = uniqueSeriesNames.IndexOf(item.Appearance.LegendText);//.Name); item.Appearance.Color = uniqueSeriesColors[colIdx]; } catch { item.Appearance.Color = "Black"; } } this.timeSeriesGraph1.AnalysisType = AnalysisType.WaterYears; this.timeSeriesGraph1.Series = finalSeriesCollection; if (seriesList.Count == 1) { this.timeSeriesGraph1.Title = HydrometInfoUtility.LookupSiteDescription(cbtt) + " Elevation:" + HydrometInfoUtility.LookupElevation(cbtt); } //timeSeriesGraph1.GraphSettings = GetGraphSettings(); this.timeSeriesGraph1.Draw(true); comboBoxCbtt.Text = cbttOrig; comboBoxPcode.Text = pcodeOrig; timeSeriesGraph1.GraphSettings = GetGraphSettings(); } finally { Cursor = Cursors.Default; } }
private static void ProcessDate(DateTime t, Arguments args, DataTable csv, bool recentDataOnly) { var xmlFileName = DownloadAndUnzip(t); XPathDocument doc = new XPathDocument(xmlFileName); Series ob = new Series(); // air temp Series tu = new Series(); //relative humidity Console.WriteLine("Saving to " + args["output"]); foreach (DataRow row in csv.Rows) { var cbtt = row["cbtt"].ToString(); if (cbtt.Trim() == "") { continue; } var inl_id = row["inl_id"].ToString(); var inel_element = row["inel_element"].ToString(); var hydromet_pcode = row["hydromet_pcode"].ToString(); Console.Write(inl_id + " " + inel_element + "(" + hydromet_pcode + ")"); //var s = ParseXmlData(doc,"ABE","ws",t); var s = ParseXmlData(doc, inl_id, inel_element, t); s.Parameter = hydromet_pcode; s.SiteID = cbtt; if (hydromet_pcode.ToLower() == "pi") { s = s * 0.01; } if (inel_element.ToLower() == "sr" || inel_element.ToLower() == "sr2") { //0.0860 W/m2 = 1 Langley s = s * 0.0860; // divide by 12 to convert to hourly rate (5 minute data) } if (hydromet_pcode.ToLower() == "ob") { ob = s; } if (hydromet_pcode.ToLower() == "tu") { tu = s; } if (recentDataOnly && t.Date == DateTime.Now.Date) {// only load the last 4 hours when reading the current date Console.WriteLine("only saving last 4 hours of data"); s.Trim(DateTime.Now.AddHours(-4), DateTime.Now.AddHours(2)); } HydrometInstantSeries.WriteToHydrometFile(s, cbtt, hydromet_pcode, "inl", args["output"], true); var tp = DewPointCalculation(ob, tu, cbtt); if (tp.Count > 0) { HydrometInstantSeries.WriteToHydrometFile(tp, cbtt, "TP", "", args["output"], true); } } }
private void AddStatistics(SeriesList wyList) { bool anyStats = checkBoxMax.Checked || checkBoxMin.Checked || checkBoxAvg.Checked || checkBoxPctls.Checked; if (!anyStats) { return; } int y1 = 1990; int y2 = 2011; int[] pctls = new int[] { }; int.TryParse(this.textBoxWY1.Text, out y1); int.TryParse(this.textBoxWY2.Text, out y2); if (checkBoxPctls.Checked) { try { string values = textBoxPctls.Text; string[] tokens = values.Split(','); pctls = Array.ConvertAll <string, int>(tokens, int.Parse); } catch { pctls = new int[] { 10, 50, 90 }; } } DateTime t1, t2; if (cySelected) { t1 = new DateTime(y1, 1, 1); t2 = new DateTime(y2, 12, 31); } else { t1 = new DateTime(y1 - 1, 10, 1); t2 = new DateTime(y2, 9, 30); } var server = HydrometInfoUtility.HydrometServerFromPreferences(); Series s; if (this.checkBoxUseInstant.Checked) { s = new HydrometInstantSeries(comboBoxCbtt.Text.Trim(), DeterminePcode(), server); } else { s = new HydrometDailySeries(comboBoxCbtt.Text.Trim(), DeterminePcode(), server); } s.Read(t1, t2); s.RemoveMissing(); s.Appearance.LegendText = ""; YearRange yr; if (cySelected) { yr = new YearRange(2000, 1); } else { yr = new YearRange(2000, 10); } var list = Math.SummaryHydrograph(s, pctls, yr.DateTime1, checkBoxMax.Checked, checkBoxMin.Checked, checkBoxAvg.Checked, false); //, false); wyList.Add(list); }
private void ProcessFile(string fileName) { Logger.WriteLine("ProcessFile('" + fileName + "')"); string importTag = "import"; // used to make friendly export filename try { TextFile tf = new TextFile(fileName); SeriesList sl = new SeriesList(); if (HydrometInstantSeries.IsValidDMS3(tf)) { importTag = "decodes"; sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf); } else if (HydrometDailySeries.IsValidArchiveFile(tf)) { importTag = "htools"; sl = HydrometDailySeries.HydrometDailyDataToSeriesList(tf); } else if (LoggerNetFile.IsValidFile(tf)) { LoggerNetFile lf = new LoggerNetFile(tf); if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0) { importTag = lf.SiteName; sl = lf.ToSeries(validPcodes); } } else if (CsvScadaFile.IsValidFile(tf)) { importTag = "scada"; m_db.GetSiteCatalog(); var scada = new CsvScadaFile(fileName); sl = scada.ToSeries(); } else if (XConnectTextFile.IsValidFile(tf)) { importTag = "xc"; var xc = new XConnectTextFile(tf); sl = xc.ToSeries(); } else if (HydrometMonthlySeries.IsValidFile(tf)) { sl = HydrometMonthlySeries.FileToSeriesList(tf); } else { Logger.WriteLine("skipped Unknown File Format: " + fileName); return; } if (m_importer == null) { m_importer = new TimeSeriesImporter(m_db, m_saveOption); } Console.WriteLine("Found " + sl.Count + " series in " + fileName); foreach (var item in sl) { Logger.WriteLine(item.Table.TableName); } if (sl.Count > 0) { m_importer.Import(sl, m_computeDependencies, m_computeDailyDependencies, importTag); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "attic", fileName); } } catch (Exception ex) { Logger.WriteLine("Error:" + ex.Message); Console.WriteLine("Error: skipping file, will move to error subdirectory " + fileName); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "error", fileName); } }
/// <summary> /// If Decodes software is installed locally /// DECODE raw data for a single parameter /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void AdvancedRawData_Click(object sender, EventArgs e) { // determine siteid (cbtt) and pcode SpreadsheetRange r = new SpreadsheetRange(wbView.RangeSelection); var col = r.SelectedRangeColumnNames[0]; var tokens = col.Trim().Split(' '); if (tokens.Length != 2) { return; } var cbtt = tokens[0].ToLower(); var pcode = tokens[1].ToLower(); // find date range that is selected. var t = r.SelectedDateRange; // account for timezone offset and transmission time delay // summer UTC-6h // winter UTC-7h var t1 = t.DateTime1.AddHours(-24); var t2 = t.DateTime2.AddHours(+24); var svr = Database.GetServer("hydromet_opendcs"); if (svr == null) { MessageBox.Show("Error connecting to the database. Please check your password"); return; } var fn = FileUtility.GetSimpleTempFileName(".txt"); var log = FileUtility.GetSimpleTempFileName(".txt"); // run DECODES to create output file DecodesUtility.RunDecodesRoutingSpec(svr, "hydromet-tools", t1, t2, cbtt, fn, log); //Don't Go Karl! foreach (var item in File.ReadAllLines(log)) { Logger.WriteLine(item); } foreach (var item in File.ReadAllLines(fn)) { Logger.WriteLine(item); } TextFile tf = new TextFile(fn); if (!HydrometInstantSeries.IsValidDMS3(tf)) { MessageBox.Show("Error reading Decodes output"); return; } // Read Decodes output var sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf); // filter by cbtt and pcode var s = sl.Find(x => x.Table.TableName == "instant_" + cbtt + "_" + pcode); if (s == null) { Logger.WriteLine("Error: could not find decoded data for " + cbtt + "/" + pcode); return; } // use dataview for sorted data // filter by date range Series decoded = s.Clone(); for (int i = 0; i < s.Count; i++) { var pt = s[i]; if (pt.DateTime >= t.DateTime1 && pt.DateTime <= t.DateTime2) { decoded.Add(pt); } } // put values into hydromet tools r.InsertSeriesValues(decoded); }
public double GetValue(string cbtt, string pcode, DateTime t) { var s = new HydrometInstantSeries(cbtt, pcode,HydrometHost.Yakima); //DateTime th = new DateTime(t.Year, t.Month, t.Day, hour, 0, 0); s.Read(t,t); if( s.Count >0 && !s[0].IsMissing) { return s[0].Value; } return Point.MissingValueFlag; }
private void ProcessFile(RouteOptions routing, string fileName) { string importTag = "import"; // used to make friendly export filename try { TextFile tf = new TextFile(fileName); SeriesList sl = new SeriesList(); if (HydrometInstantSeries.IsValidDMS3(tf)) { importTag = "decodes"; sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf); } else if (HydrometDailySeries.IsValidArchiveFile(tf)) { importTag = "htools"; sl = HydrometDailySeries.HydrometDailyDataToSeriesList(tf); } else if (LoggerNetFile.IsValidFile(tf)) { LoggerNetFile lf = new LoggerNetFile(tf); if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0) { importTag = lf.SiteName; sl = lf.ToSeries(validPcodes); } } //else if (DecodesRawFile.IsValidFile(tf)) //{ // DecodesRawFile df = new DecodesRawFile(tf); // importTag = "raw"; // sl = df.ToSeries(); //} else { Logger.WriteLine("skipped Unknown File Format: " + fileName); return; } m_importer = new TimeSeriesImporter(m_db, routing, m_saveOption); Console.WriteLine("Found " + sl.Count + " series in " + fileName); foreach (var item in sl) { Logger.WriteLine(item.Table.TableName); } if (sl.Count > 0) { m_importer.Import(sl, m_computeDependencies, m_computeDailyDependencies, importTag); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "attic", fileName); } } catch (Exception ex) { Logger.WriteLine("Error:" + ex.Message); Console.WriteLine("Error: skipping file, will move to error subdirectory " + fileName); FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "error", fileName); } }
//try to take the code from above and make it into here for the show cmd private static string Get(string s, DateTime t1, DateTime t2) { string[] cbtt = { "" }; string[] pcode = { "" }; string Id = ""; string value = ""; string pc = ""; var cfg = new Config(s, t1, t2); if (s.Trim().ToLower()[0] == 'g') { cbtt = cfg.cbtt; for (int i = 0; i < cbtt.Length; i++) { Id = LoopCBTT(cbtt[i]); if (String.IsNullOrEmpty(Id) == true) { Console.WriteLine("%W-Dayfile, no data found for get request:" + cbtt[i]); } else { string[] ID = Id.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); //improving the speed of the program var cache = new HydrometDataCache(); cache.Add(ID, t1, t2, HydrometHost.PNLinux, Reclamation.TimeSeries.TimeInterval.Irregular); HydrometInstantSeries.Cache = cache; HydrometInstantSeries.KeepFlaggedData = true; if (cfg.IsGetAll) { HydrometInstantSeries ts = new HydrometInstantSeries(ID[0].Split(' ')[0], ID[0].Split(' ')[1]); ts.Read(t1, t2); var count = (ts.MaxDateTime - ts.MinDateTime).TotalMinutes; for (int t = 0; t <= count / 15; t++) { pc = cbtt[i].ToUpper() + " " + t1.AddMinutes(t * 15).ToString("yyMMMdd"); for (int j = 0; j < ID.Length; j++) { var c = ID[j].Split(' ')[0]; var p = ID[j].Split(' ')[1]; HydrometInstantSeries.KeepFlaggedData = true; ts = new HydrometInstantSeries(c, p); ts.Read(t1, t2); pc = pc + " " + p.ToUpper(); if (j == 0) { value = " " + ts.MinDateTime.AddMinutes(t * 15).TimeOfDay + " " + ts[ts.MinDateTime.AddMinutes(t * 15)].Value; } else { value = value + " " + ts[ts.MinDateTime.AddMinutes(t * 15)].Value; } } if (ts.MinDateTime.AddMinutes(t * 15).TimeOfDay.TotalHours == 0 || ts.MinDateTime.AddMinutes(t * 15).TimeOfDay.TotalHours == 12) { WriteLines(pc); } WriteLines(value); } } if (cfg.IsGet || (cfg.IsGetAll == false && cfg.IsGetPcode)) { if (cfg.IsGetAll == false && cfg.IsGetPcode) { pcode = cfg.pcode; Id = ""; for (int j = 0; j < pcode.Length; j++) { Id = Id + cbtt[i] + " " + pcode[j] + ","; } ID = Id.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); } var count = (t2 - t1).TotalDays; for (int t = 0; t <= count; t++) { pc = cbtt[i].ToUpper() + " " + t1.AddDays(t).ToString("yyMMMdd"); for (int j = 0; j < ID.Length; j++) { var c = ID[j].Split(' ')[0]; var p = ID[j].Split(' ')[1]; HydrometInstantSeries.KeepFlaggedData = true; HydrometInstantSeries ts = new HydrometInstantSeries(c, p); ts.Read(t1, t2); pc = pc + " " + p.ToUpper(); if (j == 0) { value = " " + ts.MaxDateTime.AddDays(-count + t).TimeOfDay + " " + ts[ts.MaxDateTime.AddDays(-count + t)].Value; } else { value = value + " " + ts[ts.MaxDateTime.AddDays(-count + t)].Value; } } WriteLines(pc); WriteLines(value); } } } } } return(s); }