public TestDeleteDatabaseProperties()
{
string path;
if (LinuxUtility.IsLinux())
{
path = "/tmp";
}
else
{
path = "C:\\Temp\\";
}
string fn = Path.Combine(path, "factory.pdb");
FileUtility.GetTempFileNameInDirectory(path, ".pdb");
SQLiteServer.CreateNewDatabase(fn);
SQLiteServer svr = new SQLiteServer(fn);
db = new TimeSeriesDatabase(svr, false);
string dataPath = TestData.DataPath;
Series s = new Series("Glomma River");
db.AddSeries(s);
s = db.GetSeriesFromName("Glomma River");
s.TimeSeriesDatabase = db;
s.Properties.Set("elevation", " 690 m");
s.Properties.Save();
}
public void ExportDatabase()
{
string path;
if (LinuxUtility.IsLinux())
{
path = "/tmp";
}
else
{
path = "C:\\Temp\\";
}
db.Export(Path.Combine(path, "export1"));
}
//todo run command line function
private static string[] RunExecutable(string exe, string args)
{
if (LinuxUtility.IsLinux())
{
return(RunLocal(exe, args));
}
else
{// run remote
var tokens = File.ReadAllLines(@"c:\utils\linux\dectalk.txt");
SshClient ssh = new SshClient("dectalk", tokens[0], tokens[1]);
ssh.Connect();
var cmd = ssh.RunCommand(exe + " " + args);
Console.WriteLine(cmd.Result);
return(cmd.Result.Split('\n'));
}
}
private int SetupNetworkList(string name)
{
var rows = decodes.networklist.Select("name = '" + name + "'");
if (rows.Length == 1)
{
return(Convert.ToInt32(rows[0]["id"]));
}
int id = NextID(svr, "networklist");
decodes.networklist.AddnetworklistRow(id, name, "goes", "cbtt",
LinuxUtility.ToUnixTime(DateTime.Now) * 1000);
svr.SaveTable(decodes.networklist);
return(id);
}
public TimeSeriesDatabaseTest()
{
string path;
if (LinuxUtility.IsLinux())
{
path = "/tmp";
}
else
{
path = "C:\\Temp\\";
}
tmpDir = Path.Combine(path, "db_test");
if (!Directory.Exists(tmpDir))
{
Directory.CreateDirectory(tmpDir);
}
string fn = Path.Combine(path, "factory.pdb");
FileUtility.GetTempFileNameInDirectory(path, ".pdb");
SQLiteServer.CreateNewDatabase(fn);
SQLiteServer svr = new SQLiteServer(fn);
db = new TimeSeriesDatabase(svr, false);
//string dataPath = ReclamationTesting.Properties.Settings.Default.DataPath;
string dataPath = TestData.DataPath;
File.Copy(Path.Combine(dataPath, textFileName), Path.Combine(tmpDir, textFileName), true);
textFileName = Path.Combine(tmpDir, textFileName);
// Add some data for export test
Series s;
int c;
int sdi;
AddTextSeries(out s, out c, out sdi);
}
internal static DateTime ConvertToMountainTime(DateTime utc)
{
DateTime mountain;
if (LinuxUtility.IsLinux())
{
TimeZoneInfo tzi = TimeZoneInfo.FindSystemTimeZoneById("US/Mountain");
Console.WriteLine("try ConvertTimeFromUtc");
mountain = TimeZoneInfo.ConvertTimeFromUtc(utc, tzi);
//Console.WriteLine("try other way..");
//mountain = TimeZoneInfo.ConvertTimeBySystemTimeZoneId
// (utc, "US/Mountain");
}
else
{
mountain = TimeZoneInfo.ConvertTimeBySystemTimeZoneId
(utc, "Mountain Standard Time");
}
//Console.WriteLine("{0} (UTC) = {1} Mountain time", utc, mountain);
return(mountain);
}
public void AddBug()
{
string path;
if (LinuxUtility.IsLinux())
{
path = "/tmp";
}
else
{
path = "C:\\Temp\\";
}
DateTime t1 = new DateTime(2013, 1, 1);
var t2 = t1.AddDays(365);
var fn = FileUtility.GetTempFileNameInDirectory(path, ".pdb");
SQLiteServer svr = new SQLiteServer(fn);
TimeSeriesDatabase db = new Reclamation.TimeSeries.TimeSeriesDatabase(svr, false);
var s = new HydrometDailySeries("pal", "af");
s.Name = "pal_af";
s.Read(t1, t2);
db.AddSeries(s);
var cs = new CalculationSeries("add_test");
cs.Expression = "pal_af + pal_af";
db.AddSeries(cs);
cs = db.GetSeriesFromName("add_test") as CalculationSeries;
cs.Calculate(t1, t2);
cs = db.GetSeriesFromName("add_test") as CalculationSeries;
cs.Read();
Assert.IsTrue(cs.Count > 0);
}
public static string GetOutgoingFileName(string prefix, string cbtt, string pcode, string route = "")
{
string outgoing = ConfigurationManager.AppSettings["outgoing"];
if (outgoing == "" || outgoing == null)
{
Console.WriteLine("Error: 'outgoing' directory not defined in config file");
Logger.WriteLine("Error: 'outgoing' directory not defined in config file");
// hack hack hack
if (LinuxUtility.IsLinux())
{
outgoing = "/tmp";
}
else
{
outgoing = "C:\\Temp\\";
}
}
if (!Directory.Exists(outgoing) || outgoing == null)
{
Console.WriteLine("Error: path does not exist: '" + outgoing + "'");
}
// route is subdirectory i.e. /home/hydromet/outgoing/route
if (route != "")
{
outgoing = Path.Combine(outgoing, route);
}
if (!Directory.Exists(outgoing))
{
Console.WriteLine("Error: Directoy does not exist " + outgoing);
}
return(Path.Combine(outgoing, GetUniqueFileName(outgoing, prefix, cbtt, pcode)));
}
/// <summary>
/// Path for incoming data files
/// </summary>
/// <returns></returns>
static string GetIncommingFileName(string prefix, string cbtt, string pcode, string fileExtension)
{
string incoming = ConfigurationManager.AppSettings["incoming"];
if (incoming == "" || incoming == null)
{
Console.WriteLine("Error: 'incoming' directory not defined in config file");
Logger.WriteLine("Error: 'incoming' directory not defined in config file");
// hack hack hack
// hack hack hack
if (LinuxUtility.IsLinux())
{
incoming = "/tmp";
}
else
{
incoming = "C:\\Temp\\";
}
}
return(Path.Combine(incoming, TimeSeriesTransfer.GetUniqueFileName(incoming, prefix, cbtt, pcode, fileExtension)));
}
static Series ParseXmlData(XPathDocument doc, string siteID, string inel_element, DateTime t)
{
Series s = new Series();
var nav = doc.CreateNavigator();
DateTime inlTime = t.Date; // used to increment each 5 minutes
TimeZoneInfo mst;
TimeZoneInfo mdt;
if (LinuxUtility.IsLinux())
{
//var c = TimeZoneInfo.GetSystemTimeZones();
//foreach (var item in c)
//{
// Console.WriteLine(item);
//}
mst = TimeZoneInfo.FindSystemTimeZoneById("US/Arizona"); // no daylight shift
mdt = TimeZoneInfo.FindSystemTimeZoneById("US/Mountain"); // with daylight shifting
}
else
{
mst = TimeZoneInfo.FindSystemTimeZoneById("US Mountain Standard Time"); // no daylight shift
mdt = TimeZoneInfo.FindSystemTimeZoneById("Mountain Standard Time"); // with daylight shifting
}
string ts = "";
do
{
ts = inlTime.ToString("HH:mm");
var query = "mesonet/Time[@time=\"" + ts + "\"]/tower[@id=\"" + siteID + "\"]"; // /spd";
var nodeName = inel_element;
var tag = "";
int idx = inel_element.IndexOf(".");
if (idx > 0)
{
tag = inel_element.Substring(0, idx);
query += "/" + tag;
nodeName = inel_element.Substring(idx + 1);
}
var nodes = nav.Select(query);
while (nodes.MoveNext())
{
//Console.WriteLine(ts+ "id=" + nodes.Current.GetAttribute("id", ""));
if (nodes.Current.HasChildren)
{
var children = nodes.Current.SelectChildren(XPathNodeType.All);
while (children.MoveNext())
{
var n = children.Current;
if (n.LocalName == nodeName)
{
// Console.WriteLine(n.LocalName + " = " + n.Value);
double val = 0;
if (double.TryParse(n.Value, out val))
{
//s.Add(t.Date.AddHours(time.Hour).AddMinutes(time.Minute), val, "inl");
DateTimeWithZone td = new DateTimeWithZone(inlTime, mst);
DateTime hydrometDateTime;
if (td.TryConvertToTimeZone(mdt, out hydrometDateTime))
{
if (s.IndexOf(hydrometDateTime) < 0)
{// in November time change, we have duplicate dates..
s.Add(hydrometDateTime, val, "inl");
}
else
{
Console.WriteLine("skipping dateTime " + hydrometDateTime.ToString());
}
}
}
}
else if (n.LocalName == inel_element)
{
}
}
}
}
inlTime = inlTime.AddMinutes(5);
} while (ts != "23:55");
Console.WriteLine("Read " + s.Count + " data points ");
return(s);
}
/// <summary>
/// Gets a table of hydromet data from the web. The table has a column for each parameter. I the table is
/// instant data a flag column is included to the right of each data column
/// </summary>
/// <param name="cgi"></param>
/// <param name="query"></param>
/// <param name="t1"></param>
/// <param name="t2"></param>
/// <param name="back">when greater than zero used instead of t1,t2 (hours (instant) or days(daily) back)</param>
/// <param name="endOfMonth"></param>
/// <returns></returns>
static DataTable Table(string cgi, string query, DateTime t1, DateTime t2, int back = 0, bool endOfMonth = false)
{
//if (t2 > DateTime.Now ) can't do this for 6190 (special 30 yr average year)
// t2 = DateTime.Now;
int colIndex = 1;
string flag = " ";
string payload = "parameter=" + query
+ "&syer=" + t1.Year.ToString()
+ "&smnth=" + t1.Month.ToString()
+ "&sdy=" + t1.Day.ToString()
+ "&eyer=" + t2.Year.ToString()
+ "&emnth=" + t2.Month.ToString()
+ "&edy=" + t2.Day.ToString()
+ "&format=2";
if (back > 0)
{
payload = "parameter=" + query
+ "&back=" + back
+ "&format=2";
}
string[] data = Web.GetPage(cgi, payload, HydrometInfoUtility.WebCaching);
bool hasFlag = cgi.ToLower().IndexOf("webdaycsv") >= 0 ||
cgi.IndexOf("webmpollcsv") >= 0 ||
cgi.IndexOf("monthly") >= 0 ||
cgi.IndexOf("instant") >= 0;
TextFile tf = new TextFile(data);
int idx1 = tf.IndexOf("BEGIN DATA");
int idx2 = tf.IndexOf("END DATA");
int idxDates = tf.IndexOfRegex(@"(\d{1,2}/\d{1,2}/\d{1,2})|(\s[A-Z]{3}\s+\d{4})"); // find first date
if (idx1 < 0 || idx2 < 0 || idx2 == idx1 + 1 || idxDates < 0)
{
return(new DataTable("No_data"));
}
string strTitles = "";
for (int i = idx1 + 1; i < idxDates; i++)
{
strTitles += data[i]; // put wrapped titles back in single line.
}
string[] columnNames = strTitles.Split(',');
DataTable tbl = new DataTable("hydromet");
CreateColumnNames(hasFlag, columnNames, tbl);
for (int i = idxDates; i < idx2; i++)
{
JoinWrappedLines(data, i);
string line = data[i];
if (line == "")
{
continue;
}
string[] parts = line.Trim().Split(',');
if (parts.Length == 0)
{
continue;
}
DataRow row = tbl.NewRow();
DateTime date = DateTime.Now;
//if (!DateTime.TryParse(parts[0], out date))
if (!LinuxUtility.TryParseDateTime(parts[0], out date))
{
Logger.WriteLine("Error parsing date " + parts[0]);
break;
}
if (endOfMonth)
{
date = date.Date.EndOfMonth();
}
row[0] = date;
for (int j = 1; j < parts.Length; j++)
{
string strValue = parts[j];
if (strValue.IndexOf("NO RECORD") >= 0 ||
strValue.IndexOf("MISSING") >= 0 ||
strValue.IndexOf("998877.00") >= 0 ||
strValue.Trim() == "" ||
strValue.Trim() == PointFlag.Missing
)
{
row[colIndex] = System.DBNull.Value;
if (hasFlag)
{
row[colIndex + 1] = "";
colIndex++;
}
}
else
{
//Console.WriteLine("str = "+strValue);
double val = -999;
int index = strValue.IndexOf(".");
string str = strValue;
if (hasFlag && index >= 0)
{
str = strValue.Substring(0, index + 3);
if (strValue.Length > str.Length)
{
flag = strValue.Substring(str.Length, 1);
}
else
{
flag = " ";
}
}
if (!double.TryParse(str, out val))
{
Logger.WriteLine("Error converting " + str + " to a number ");
}
try
{
row[colIndex] = val;
if (hasFlag)
{
row[colIndex + 1] = flag;
colIndex++;
}
}
catch (Exception ex)
{
Logger.WriteLine(ex.Message);
}
}
colIndex++;
}
tbl.Rows.Add(row);
colIndex = 1;
}
//DataSet ds = new DataSet();
//ds.Tables.Add(tbl);
return(tbl);
}
private void InsertGoesPlatform(int networklistid)
{
platformID = NextID(svr, "platform");
platformConfigID = NextID(svr, "platformconfig");
decodesScriptID = NextID(svr, "decodesscript");
for (int i = 0; i < mcf.sitemcf.Rows.Count; i++)
{
McfDataSet.sitemcfRow s = mcf.sitemcf[i];
//var usgs_id = "";
//if (!UsgsSite(s.SITE, out usgs_id))
// continue;
if (!InSiteList(s.SITE))
{
continue;
}
string nessid = s.NESSID.Trim();
if (s.TTYPE != "SAT" || s.MASTERSW == 0 || nessid == "" || nessid == "0" ||
decodes.transportmedium.Select("mediumid='" + nessid + "'").Length > 0 || s.PCHAN == 199)
{
Console.WriteLine("Skipping adding existing/parked platform " + s.SITE + " " + s.NESSID);
continue;
}
string cfgName = "";//
string platformDesignator = "";
string platformDescription = "";
string parameterFilter = "";
List <McfDataSet.sitemcfRow> sitesAtTransmitter =
new List <McfDataSet.sitemcfRow>(from row in mcf.sitemcf.AsEnumerable()
where row.NESSID == nessid
//&& row.NESSID != ""
//&& row.NESSID != "0"
orderby row.SITE
select row);
for (int j = 0; j < sitesAtTransmitter.Count; j++)
{
if (cfgName.Length > 0)
{
cfgName += "-";
}
cfgName += "" + sitesAtTransmitter[j].SITE.Trim();
if (platformDesignator.Length > 0)
{
platformDesignator += "-";
}
platformDesignator += sitesAtTransmitter[j].SITE.Trim();
if (platformDescription.Length > 0)
{
platformDescription += "\n ";
}
platformDescription += sitesAtTransmitter[j].DESCR;
if (parameterFilter.Length > 0)
{
parameterFilter += " or ";
}
parameterFilter += " GOESDEC like '" + sitesAtTransmitter[j].SITE.PadRight(8) + "%' ";
}
var pc = decodes.platformconfig;
var p = decodes.platform;
// build Platform and Platform Config together.
int siteID = LookupSiteIDFromCBTT(s.SITE.Trim());
var duplicates = pc.Select("name = '" + cfgName + "'");
if (duplicates.Length > 0)
{
Console.WriteLine(cfgName + " allready exists.. skipping");
continue;
}
pc.AddplatformconfigRow(platformConfigID, cfgName, s.DESCR, GetManufactureCode(s.DCPMFG));
p.AddplatformRow(platformID, GetAgencyName(s.GRP), "TRUE", siteID, platformConfigID, platformDescription,
LinuxUtility.ToUnixTime(DateTime.Now.Date) * 1000,
LinuxUtility.ToUnixTime(DateTime.Now.AddYears(100)) * 1000, platformDesignator);
decodes.transportmedium.AddtransportmediumRow(platformID, "goes-self-timed", nessid, "ST",
s.PCHAN, s.REPTIM, s.TSSIZ, s.REPINT * 60, GetManufactureCode(s.DCPMFG),
0, "S", "UTC", "", 0, 0, "U", 0, "FALSE", "", "");
decodes.networklistentry.AddnetworklistentryRow(networklistid, nessid, s.SITE.Trim(), s.DESCR);
decodes.platformproperty.AddplatformpropertyRow(platformID, "HydrometGroup", getHydrometCatagoryName(s));
decodes.platformproperty.AddplatformpropertyRow(platformID, "USGS", "True");
decodes.platformproperty.AddplatformpropertyRow(platformID, "expectLength", s.SMSGLEN.ToString());
// need parameter filter....
var parameters = (McfDataSet.goesmcfRow[])mcf.goesmcf.Select(parameterFilter + " and ACTIVE = 1");
string dataOrder = "D";
if (s.DCPMFG == "A")
{
dataOrder = "A";
}
decodes.decodesscript.AdddecodesscriptRow(decodesScriptID, platformConfigID, "ST", "Decodes", dataOrder);
BuildDecodesScripts(sitesAtTransmitter.ToArray(), siteID, parameters, platformConfigID, decodesScriptID);
InsertBatteryVoltScript(platformConfigID, decodesScriptID, parameters.Length + 1, s);
m_converter.AddBatteryVoltConverter(decodesScriptID, parameters.Length + 1, s);
platformID++;
platformConfigID++;
decodesScriptID++;
}
}
