public TestDeleteDatabaseProperties() { string path; if (LinuxUtility.IsLinux()) { path = "/tmp"; } else { path = "C:\\Temp\\"; } string fn = Path.Combine(path, "factory.pdb"); FileUtility.GetTempFileNameInDirectory(path, ".pdb"); SQLiteServer.CreateNewDatabase(fn); SQLiteServer svr = new SQLiteServer(fn); db = new TimeSeriesDatabase(svr, false); string dataPath = TestData.DataPath; Series s = new Series("Glomma River"); db.AddSeries(s); s = db.GetSeriesFromName("Glomma River"); s.TimeSeriesDatabase = db; s.Properties.Set("elevation", " 690 m"); s.Properties.Save(); }
public void ExportDatabase() { string path; if (LinuxUtility.IsLinux()) { path = "/tmp"; } else { path = "C:\\Temp\\"; } db.Export(Path.Combine(path, "export1")); }
//todo run command line function private static string[] RunExecutable(string exe, string args) { if (LinuxUtility.IsLinux()) { return(RunLocal(exe, args)); } else {// run remote var tokens = File.ReadAllLines(@"c:\utils\linux\dectalk.txt"); SshClient ssh = new SshClient("dectalk", tokens[0], tokens[1]); ssh.Connect(); var cmd = ssh.RunCommand(exe + " " + args); Console.WriteLine(cmd.Result); return(cmd.Result.Split('\n')); } }
private int SetupNetworkList(string name) { var rows = decodes.networklist.Select("name = '" + name + "'"); if (rows.Length == 1) { return(Convert.ToInt32(rows[0]["id"])); } int id = NextID(svr, "networklist"); decodes.networklist.AddnetworklistRow(id, name, "goes", "cbtt", LinuxUtility.ToUnixTime(DateTime.Now) * 1000); svr.SaveTable(decodes.networklist); return(id); }
public TimeSeriesDatabaseTest() { string path; if (LinuxUtility.IsLinux()) { path = "/tmp"; } else { path = "C:\\Temp\\"; } tmpDir = Path.Combine(path, "db_test"); if (!Directory.Exists(tmpDir)) { Directory.CreateDirectory(tmpDir); } string fn = Path.Combine(path, "factory.pdb"); FileUtility.GetTempFileNameInDirectory(path, ".pdb"); SQLiteServer.CreateNewDatabase(fn); SQLiteServer svr = new SQLiteServer(fn); db = new TimeSeriesDatabase(svr, false); //string dataPath = ReclamationTesting.Properties.Settings.Default.DataPath; string dataPath = TestData.DataPath; File.Copy(Path.Combine(dataPath, textFileName), Path.Combine(tmpDir, textFileName), true); textFileName = Path.Combine(tmpDir, textFileName); // Add some data for export test Series s; int c; int sdi; AddTextSeries(out s, out c, out sdi); }
internal static DateTime ConvertToMountainTime(DateTime utc) { DateTime mountain; if (LinuxUtility.IsLinux()) { TimeZoneInfo tzi = TimeZoneInfo.FindSystemTimeZoneById("US/Mountain"); Console.WriteLine("try ConvertTimeFromUtc"); mountain = TimeZoneInfo.ConvertTimeFromUtc(utc, tzi); //Console.WriteLine("try other way.."); //mountain = TimeZoneInfo.ConvertTimeBySystemTimeZoneId // (utc, "US/Mountain"); } else { mountain = TimeZoneInfo.ConvertTimeBySystemTimeZoneId (utc, "Mountain Standard Time"); } //Console.WriteLine("{0} (UTC) = {1} Mountain time", utc, mountain); return(mountain); }
public void AddBug() { string path; if (LinuxUtility.IsLinux()) { path = "/tmp"; } else { path = "C:\\Temp\\"; } DateTime t1 = new DateTime(2013, 1, 1); var t2 = t1.AddDays(365); var fn = FileUtility.GetTempFileNameInDirectory(path, ".pdb"); SQLiteServer svr = new SQLiteServer(fn); TimeSeriesDatabase db = new Reclamation.TimeSeries.TimeSeriesDatabase(svr, false); var s = new HydrometDailySeries("pal", "af"); s.Name = "pal_af"; s.Read(t1, t2); db.AddSeries(s); var cs = new CalculationSeries("add_test"); cs.Expression = "pal_af + pal_af"; db.AddSeries(cs); cs = db.GetSeriesFromName("add_test") as CalculationSeries; cs.Calculate(t1, t2); cs = db.GetSeriesFromName("add_test") as CalculationSeries; cs.Read(); Assert.IsTrue(cs.Count > 0); }
public static string GetOutgoingFileName(string prefix, string cbtt, string pcode, string route = "") { string outgoing = ConfigurationManager.AppSettings["outgoing"]; if (outgoing == "" || outgoing == null) { Console.WriteLine("Error: 'outgoing' directory not defined in config file"); Logger.WriteLine("Error: 'outgoing' directory not defined in config file"); // hack hack hack if (LinuxUtility.IsLinux()) { outgoing = "/tmp"; } else { outgoing = "C:\\Temp\\"; } } if (!Directory.Exists(outgoing) || outgoing == null) { Console.WriteLine("Error: path does not exist: '" + outgoing + "'"); } // route is subdirectory i.e. /home/hydromet/outgoing/route if (route != "") { outgoing = Path.Combine(outgoing, route); } if (!Directory.Exists(outgoing)) { Console.WriteLine("Error: Directoy does not exist " + outgoing); } return(Path.Combine(outgoing, GetUniqueFileName(outgoing, prefix, cbtt, pcode))); }
/// <summary> /// Path for incoming data files /// </summary> /// <returns></returns> static string GetIncommingFileName(string prefix, string cbtt, string pcode, string fileExtension) { string incoming = ConfigurationManager.AppSettings["incoming"]; if (incoming == "" || incoming == null) { Console.WriteLine("Error: 'incoming' directory not defined in config file"); Logger.WriteLine("Error: 'incoming' directory not defined in config file"); // hack hack hack // hack hack hack if (LinuxUtility.IsLinux()) { incoming = "/tmp"; } else { incoming = "C:\\Temp\\"; } } return(Path.Combine(incoming, TimeSeriesTransfer.GetUniqueFileName(incoming, prefix, cbtt, pcode, fileExtension))); }
static Series ParseXmlData(XPathDocument doc, string siteID, string inel_element, DateTime t) { Series s = new Series(); var nav = doc.CreateNavigator(); DateTime inlTime = t.Date; // used to increment each 5 minutes TimeZoneInfo mst; TimeZoneInfo mdt; if (LinuxUtility.IsLinux()) { //var c = TimeZoneInfo.GetSystemTimeZones(); //foreach (var item in c) //{ // Console.WriteLine(item); //} mst = TimeZoneInfo.FindSystemTimeZoneById("US/Arizona"); // no daylight shift mdt = TimeZoneInfo.FindSystemTimeZoneById("US/Mountain"); // with daylight shifting } else { mst = TimeZoneInfo.FindSystemTimeZoneById("US Mountain Standard Time"); // no daylight shift mdt = TimeZoneInfo.FindSystemTimeZoneById("Mountain Standard Time"); // with daylight shifting } string ts = ""; do { ts = inlTime.ToString("HH:mm"); var query = "mesonet/Time[@time=\"" + ts + "\"]/tower[@id=\"" + siteID + "\"]"; // /spd"; var nodeName = inel_element; var tag = ""; int idx = inel_element.IndexOf("."); if (idx > 0) { tag = inel_element.Substring(0, idx); query += "/" + tag; nodeName = inel_element.Substring(idx + 1); } var nodes = nav.Select(query); while (nodes.MoveNext()) { //Console.WriteLine(ts+ "id=" + nodes.Current.GetAttribute("id", "")); if (nodes.Current.HasChildren) { var children = nodes.Current.SelectChildren(XPathNodeType.All); while (children.MoveNext()) { var n = children.Current; if (n.LocalName == nodeName) { // Console.WriteLine(n.LocalName + " = " + n.Value); double val = 0; if (double.TryParse(n.Value, out val)) { //s.Add(t.Date.AddHours(time.Hour).AddMinutes(time.Minute), val, "inl"); DateTimeWithZone td = new DateTimeWithZone(inlTime, mst); DateTime hydrometDateTime; if (td.TryConvertToTimeZone(mdt, out hydrometDateTime)) { if (s.IndexOf(hydrometDateTime) < 0) {// in November time change, we have duplicate dates.. s.Add(hydrometDateTime, val, "inl"); } else { Console.WriteLine("skipping dateTime " + hydrometDateTime.ToString()); } } } } else if (n.LocalName == inel_element) { } } } } inlTime = inlTime.AddMinutes(5); } while (ts != "23:55"); Console.WriteLine("Read " + s.Count + " data points "); return(s); }
/// <summary> /// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is /// instant data a flag column is included to the right of each data column /// </summary> /// <param name="cgi"></param> /// <param name="query"></param> /// <param name="t1"></param> /// <param name="t2"></param> /// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param> /// <param name="endOfMonth"></param> /// <returns></returns> static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false) { //if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year) // t2 = DateTime.Now; int colIndex = 1; string flag = " "; string payload = "parameter=" + query + "&syer=" + t1.Year.ToString() + "&smnth=" + t1.Month.ToString() + "&sdy=" + t1.Day.ToString() + "&eyer=" + t2.Year.ToString() + "&emnth=" + t2.Month.ToString() + "&edy=" + t2.Day.ToString() + "&format=2"; if (back > 0) { payload = "parameter=" + query + "&back=" + back + "&format=2"; } string[] data = Web.GetPage(cgi, payload, HydrometInfoUtility.WebCaching); bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0 || cgi.IndexOf("webmpollcsv") >= 0 || cgi.IndexOf("monthly") >= 0 || cgi.IndexOf("instant") >= 0; TextFile tf = new TextFile(data); int idx1 = tf.IndexOf("BEGIN DATA"); int idx2 = tf.IndexOf("END DATA"); int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date if (idx1 < 0 || idx2 < 0 || idx2 == idx1 + 1 || idxDates < 0) { return(new DataTable("No_data")); } string strTitles = ""; for (int i = idx1 + 1; i < idxDates; i++) { strTitles += data[i]; // put wrapped titles back in single line. } string[] columnNames = strTitles.Split(','); DataTable tbl = new DataTable("hydromet"); CreateColumnNames(hasFlag, columnNames, tbl); for (int i = idxDates; i < idx2; i++) { JoinWrappedLines(data, i); string line = data[i]; if (line == "") { continue; } string[] parts = line.Trim().Split(','); if (parts.Length == 0) { continue; } DataRow row = tbl.NewRow(); DateTime date = DateTime.Now; //if (!DateTime.TryParse(parts[0], out date)) if (!LinuxUtility.TryParseDateTime(parts[0], out date)) { Logger.WriteLine("Error parsing date " + parts[0]); break; } if (endOfMonth) { date = date.Date.EndOfMonth(); } row[0] = date; for (int j = 1; j < parts.Length; j++) { string strValue = parts[j]; if (strValue.IndexOf("NO RECORD") >= 0 || strValue.IndexOf("MISSING") >= 0 || strValue.IndexOf("998877.00") >= 0 || strValue.Trim() == "" || strValue.Trim() == PointFlag.Missing ) { row[colIndex] = System.DBNull.Value; if (hasFlag) { row[colIndex + 1] = ""; colIndex++; } } else { //Console.WriteLine("str = "+strValue); double val = -999; int index = strValue.IndexOf("."); string str = strValue; if (hasFlag && index >= 0) { str = strValue.Substring(0, index + 3); if (strValue.Length > str.Length) { flag = strValue.Substring(str.Length, 1); } else { flag = " "; } } if (!double.TryParse(str, out val)) { Logger.WriteLine("Error converting " + str + " to a number "); } try { row[colIndex] = val; if (hasFlag) { row[colIndex + 1] = flag; colIndex++; } } catch (Exception ex) { Logger.WriteLine(ex.Message); } } colIndex++; } tbl.Rows.Add(row); colIndex = 1; } //DataSet ds = new DataSet(); //ds.Tables.Add(tbl); return(tbl); }
private void InsertGoesPlatform(int networklistid) { platformID = NextID(svr, "platform"); platformConfigID = NextID(svr, "platformconfig"); decodesScriptID = NextID(svr, "decodesscript"); for (int i = 0; i < mcf.sitemcf.Rows.Count; i++) { McfDataSet.sitemcfRow s = mcf.sitemcf[i]; //var usgs_id = ""; //if (!UsgsSite(s.SITE, out usgs_id)) // continue; if (!InSiteList(s.SITE)) { continue; } string nessid = s.NESSID.Trim(); if (s.TTYPE != "SAT" || s.MASTERSW == 0 || nessid == "" || nessid == "0" || decodes.transportmedium.Select("mediumid='" + nessid + "'").Length > 0 || s.PCHAN == 199) { Console.WriteLine("Skipping adding existing/parked platform " + s.SITE + " " + s.NESSID); continue; } string cfgName = "";// string platformDesignator = ""; string platformDescription = ""; string parameterFilter = ""; List <McfDataSet.sitemcfRow> sitesAtTransmitter = new List <McfDataSet.sitemcfRow>(from row in mcf.sitemcf.AsEnumerable() where row.NESSID == nessid //&& row.NESSID != "" //&& row.NESSID != "0" orderby row.SITE select row); for (int j = 0; j < sitesAtTransmitter.Count; j++) { if (cfgName.Length > 0) { cfgName += "-"; } cfgName += "" + sitesAtTransmitter[j].SITE.Trim(); if (platformDesignator.Length > 0) { platformDesignator += "-"; } platformDesignator += sitesAtTransmitter[j].SITE.Trim(); if (platformDescription.Length > 0) { platformDescription += "\n "; } platformDescription += sitesAtTransmitter[j].DESCR; if (parameterFilter.Length > 0) { parameterFilter += " or "; } parameterFilter += " GOESDEC like '" + sitesAtTransmitter[j].SITE.PadRight(8) + "%' "; } var pc = decodes.platformconfig; var p = decodes.platform; // build Platform and Platform Config together. int siteID = LookupSiteIDFromCBTT(s.SITE.Trim()); var duplicates = pc.Select("name = '" + cfgName + "'"); if (duplicates.Length > 0) { Console.WriteLine(cfgName + " allready exists.. skipping"); continue; } pc.AddplatformconfigRow(platformConfigID, cfgName, s.DESCR, GetManufactureCode(s.DCPMFG)); p.AddplatformRow(platformID, GetAgencyName(s.GRP), "TRUE", siteID, platformConfigID, platformDescription, LinuxUtility.ToUnixTime(DateTime.Now.Date) * 1000, LinuxUtility.ToUnixTime(DateTime.Now.AddYears(100)) * 1000, platformDesignator); decodes.transportmedium.AddtransportmediumRow(platformID, "goes-self-timed", nessid, "ST", s.PCHAN, s.REPTIM, s.TSSIZ, s.REPINT * 60, GetManufactureCode(s.DCPMFG), 0, "S", "UTC", "", 0, 0, "U", 0, "FALSE", "", ""); decodes.networklistentry.AddnetworklistentryRow(networklistid, nessid, s.SITE.Trim(), s.DESCR); decodes.platformproperty.AddplatformpropertyRow(platformID, "HydrometGroup", getHydrometCatagoryName(s)); decodes.platformproperty.AddplatformpropertyRow(platformID, "USGS", "True"); decodes.platformproperty.AddplatformpropertyRow(platformID, "expectLength", s.SMSGLEN.ToString()); // need parameter filter.... var parameters = (McfDataSet.goesmcfRow[])mcf.goesmcf.Select(parameterFilter + " and ACTIVE = 1"); string dataOrder = "D"; if (s.DCPMFG == "A") { dataOrder = "A"; } decodes.decodesscript.AdddecodesscriptRow(decodesScriptID, platformConfigID, "ST", "Decodes", dataOrder); BuildDecodesScripts(sitesAtTransmitter.ToArray(), siteID, parameters, platformConfigID, decodesScriptID); InsertBatteryVoltScript(platformConfigID, decodesScriptID, parameters.Length + 1, s); m_converter.AddBatteryVoltConverter(decodesScriptID, parameters.Length + 1, s); platformID++; platformConfigID++; decodesScriptID++; } }