/// <summary>
/// Lookup hydromet Series.
/// name is interval_cbtt_pcode
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public override ParserResult Lookup(string name, TimeInterval defaultInterval)
{
var interval = defaultInterval;
TimeSeriesName tn = new TimeSeriesName(name);
if( tn.HasInterval)
{
interval = tn.GetTimeInterval();
}
if (tn.Valid)
{
Logger.WriteLine("Hydromet Lookup " + tn.siteid + "," + tn.pcode);
var s = new Series();
if (interval == TimeInterval.Monthly)
{
s = new HydrometMonthlySeries(tn.siteid, tn.pcode,svr);
}
else if (interval == TimeInterval.Irregular)
{
s = new HydrometInstantSeries(tn.siteid, tn.pcode,svr);
}
else if (interval == TimeInterval.Daily)
{
s = new HydrometDailySeries(tn.siteid, tn.pcode,svr);
}
return new ParserResult(s);
}
else
{
return base.Lookup(name,interval);
}
}
private double Lookup30YearAverageRunoff(DateTime t)
{
var tokens = TextFile.Split(aveargeFlowStation);
if (aveargeFlowStation == "" || tokens.Length != 2)
{
return(0);
}
return(HydrometMonthlySeries.AverageValue30Year(tokens[0], tokens[1], t.Month, YTerm.Month2));
}
public void MonthlyMissingRecords()
{
HydrometMonthlySeries s = new HydrometMonthlySeries("jkpi", "se");
DateTime t1 = new DateTime(3000, 10, 1);
DateTime t2 = new DateTime(3001, 9, 1);
s.Read(t1, t2);
s.WriteToConsole();
Assert.AreEqual(12, s.Count);
Assert.AreEqual(12, s.CountMissing());
}
void ReadMonthlyForecast(DateTime t1, DateTime t2)
{
fc = new HydrometMonthlySeries(controlPoint.StationQU, "FC");// hgh "fms"
fc.TimePostion = TimePostion.FirstOfMonth;
fcm = new HydrometMonthlySeries(controlPoint.StationQU, "FCM");
fcm.TimePostion = TimePostion.MidMonth;
fc.Read(t1, t2);
fcm.Read(t1, t2);
}
public void MonthlyMissingRecords()
{
HydrometMonthlySeries s = new HydrometMonthlySeries("jkpi", "se");
DateTime t1 = new DateTime(3000, 10, 1);
DateTime t2 = new DateTime(3001, 9, 1);
s.Read(t1, t2);
s.WriteToConsole();
Assert.AreEqual(12, s.Count);
Assert.AreEqual(12, s.CountMissing());
}
/// <summary>
/// Returns a list of cbtt pode pairs.
/// used to pre-load data using HydrometMonthlyDataCache
/// </summary>
/// <returns></returns>
public List <string> GetCbttPcodeList(bool avgParameter = false)
{
var rval = new List <string>();
foreach (var x in XTerms)
{
rval.AddRange(x.cbttPodes);
}
rval.AddRange(YTerm.cbttPodes);
// check for calculations (where cbtt_pcode) format is used.
// example JCK_AF
for (int i = 0; i < rval.Count; i++)
{
string pattern = "(?<cbttPcode>[A-Z]{2,8}_[A-Z]{2,8})";
var mc = Regex.Matches(rval[i], pattern);
if (mc.Count > 0)
{
rval.RemoveAt(i);
foreach (Match item in mc)
{
rval.Add(item.Groups["cbttPcode"].Value.Replace("_", " "));
}
}
}
rval = rval.Distinct().ToList();
if (avgParameter)
{// modify parameter codes to represent average.
List <string> a = new List <string>();
for (int i = 0; i < rval.Count; i++)
{
var tokens = rval[i].Split(' ');
string cbtt = tokens[0];
string pcode = tokens[1];
pcode = HydrometMonthlySeries.LookupAveargePcode(pcode);
if (pcode != "")
{
a.Add(cbtt + " " + pcode);
}
}
return(a);
}
return(rval);
}
public static Series GetLatestForecast(string cbtt, Int32 waterYear)
{
var pc = "fc";
if (cbtt.ToLower() == "hgh")
{
pc = "fms";
}
Series forecast = new HydrometMonthlySeries(cbtt, pc);
var t1 = new DateTime(waterYear, 1, 1);
var t2 = new DateTime(waterYear, 7, 1);
forecast.Read(t1, t2);
return(forecast);
}
private static bool KeepThisSeries(TimeSeriesDatabaseDataSet.sitecatalogDataTable sitecatalog,
string siteid, string pcode, string years)
{
HydrometMonthlySeries m = new HydrometMonthlySeries(siteid, pcode, HydrometHost.PN);
var goodYears = Regex.IsMatch(years, "20[0-9]{2}");
var rows = sitecatalog.Select("siteid='" + siteid + "'");
if (rows.Length == 0 && m.Name != "")
{
//Console.WriteLine("siteid could be added to catalog:'"+siteid+"' "+m.Name);
}
return(rows.Length == 1 && goodYears);
}
public WodiRuleCurve(string cbtt, int waterYear, FillType fType)
: base("wodi.space", fType)
{
this.waterYear = waterYear;
qu = new HydrometDailySeries("wodi", "qu");
var t1 = new DateTime(waterYear - 1, 10, 1);
var t2 = new DateTime(waterYear, 9, 30);
qu.Read(t1, t2);
tblQu = FcPlotDataSet.GetTable("wodi.inflow");
tblfc = FcPlotDataSet.GetTable("wodi.forecast");
fc = new HydrometMonthlySeries("wodi", "fc");
fc.TimePostion = TimePostion.EndOfMonth;
fc.Read(t1, t2);
ReadQUAverage();
}
void ReadMonthlyForecast(DateTime t1, DateTime t2)
{
var pc = "fc";
if (controlPoint.StationFC.ToLower() == "hgh")
{
pc = "fms";
}
fc = new HydrometMonthlySeries(controlPoint.StationFC, pc);// hgh "fms"
fc.TimePostion = TimePostion.FirstOfMonth;
fcm = new HydrometMonthlySeries(controlPoint.StationFC, "FCM");
fcm.TimePostion = TimePostion.MidMonth;
fc.Read(t1, t2);
fcm.Read(t1, t2);
}
private string LookupUnits(string pcode)
{
if (m_db == TimeInterval.Daily)
{
return(HydrometInfoUtility.LookupDailyUnits(pcode));
}
if (m_db == TimeInterval.Irregular)
{
return(HydrometInfoUtility.LookupDayfileUnits(pcode));
}
if (m_db == TimeInterval.Monthly)
{
return(HydrometMonthlySeries.LookupUnits(pcode));
}
return("");
}
public void Monthly30yrAverage()
{
HydrometMonthlySeries s = new HydrometMonthlySeries("jkpi", "sem");
DateTime t1 = new DateTime(6189, 10, 1);
DateTime t2 = new DateTime(6190, 9, 1);
double[] sem = { 0, 3.5, 9.6, 16.6, 23.0, 28.8, 33.3, 23.0, 0, 0, 0, 0 };
s.Read(t1, t2);
Assert.AreEqual(12, s.Count);
DateTime t = t1;
for (int i = 0; i < sem.Length; i++)
{
Assert.AreEqual(sem[i], s[i].Value);
Assert.AreEqual(t.Month, s[i].DateTime.Month);
Assert.AreEqual(t.Year, s[i].DateTime.Year);
Assert.AreEqual("S", s[i].Flag);
t = t.AddMonths(1);
}
//m.CreateSeries(row);
}
protected SeriesList CreateSeriesList(bool getAveragePcode = false)
{
list = new SeriesList();
for (int i = 0; i < cbttPodes.Count; i++)
{
var tokens = cbttPodes[i].Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (tokens.Length != 2)
{
if (cbttPodes[i].IndexOf("_") >= 0)
{
// QU or other calculation
var s = new CalculationSeries();
s.Expression = cbttPodes[i];
s.TimeInterval = TimeInterval.Monthly;
s.Parser.VariableResolver = HydrometData.GetVariableResolver();
list.Add(s);
}
else
{
throw new FormatException(cbttPodes[i]);
}
}
else
{
var cbtt = tokens[0].Trim();
var pcode = tokens[1].Trim();
HydrometHost host = HydrometData.s_server;
if (getAveragePcode)
{ // used for snow/precip
throw new NotImplementedException("Error. Legacy System retired.");
// pcode = HydrometMonthlySeries.LookupAveargePcode(pcode);
// host = HydrometHost.PN;
}
var s = new HydrometMonthlySeries(cbtt, pcode, host);
list.Add(s);
}
}
return(list);
}
/// <summary>
/// Hydromet mpoll stores the average for water years 1960-1989
/// in the year 6189
/// </summary>
//[Test]
public void Monthly30yrAverage()
{
HydrometMonthlySeries s = new HydrometMonthlySeries("jkpi", "sem");
DateTime t1 = new DateTime(6189, 10, 1);
DateTime t2 = new DateTime(6190, 9, 1);
double[] sem = { 0, 3.5, 9.6, 16.6, 23.0, 28.8, 33.3, 23.0, 0, 0, 0, 0 };
s.Read(t1, t2);
Assert.AreEqual(12, s.Count);
DateTime t = t1;
for (int i = 0; i < sem.Length; i++)
{
Assert.AreEqual(sem[i], s[i].Value);
Assert.AreEqual(t.Month, s[i].DateTime.Month);
Assert.AreEqual(t.Year, s[i].DateTime.Year);
Assert.AreEqual("S", s[i].Flag);
t = t.AddMonths(1);
}
//m.CreateSeries(row);
}
/// <summary>
/// Estimates future data using averages stored in the monthly database
/// used by AntecedentRunoff Term and Precipitation Term
/// </summary>
/// <param name="list"></param>
/// <param name="forecastDate"></param>
internal static void EstimateFutureWithAverage(SeriesList list, DateTime forecastDate, double estimationScaleFactor)
{
// filter out CalculationSeries
var subList = RemoveCalculationSeries(list);
foreach (var s in subList)
{
for (int dateIndex = 0; dateIndex < s.Count; dateIndex++)
{
var mp = s[dateIndex];
if (mp.DateTime.FirstOfMonth() >= forecastDate.FirstOfMonth()) //... BETTER VERSION???? needed for mid month
//if (mp.DateTime >= forecastDate)
{
string cbtt = s.ConnectionStringToken("cbtt");
string pcode = s.ConnectionStringToken("pcode");
var avg = HydrometMonthlySeries.ReadAverageValue(cbtt, pcode, mp.DateTime);
if (!avg.IsMissing)
{
if (forecastDate.Day != 1 && forecastDate.Month == mp.DateTime.Month &&
!mp.IsMissing && mp.Flag == "M") // mid-month forecast estimate using partial month
{
double numDays = DateTime.DaysInMonth(avg.DateTime.Year, avg.DateTime.Month);
double proration = (numDays - forecastDate.Day) / numDays;
avg.Value = avg.Value * proration + mp.Value;
s[dateIndex] = avg;
}
else // use average value (with scale factor)
{
avg.Value = avg.Value * estimationScaleFactor;
s[dateIndex] = avg;
Console.WriteLine("using estimated data " + cbtt + " " + pcode + " " + s[dateIndex].DateTime.ToString("yyyy MMM"));
}
}
}
}
}
}
public void TestHydrometResidualForecast()
{
DateTime t1 = new DateTime(2014, 2, 1);
DateTime t2 = new DateTime(2014, 2, 28);
var qu = new HydrometDailySeries("prvo", "qu");
var fc = new HydrometMonthlySeries("prvo", "fc");
var fcm = new HydrometMonthlySeries("prvo", "fcm");
var residual = new HydrometDailySeries("prv", "fcresid");
Assert.AreEqual(TimeInterval.Daily, residual.TimeInterval);
qu.Read(t1, t2);
fc.Read(t1, t2);
fcm.Read(t1, t2);
var forecast = Math.HydrometForecastMonthlyToDaily(fc, fcm);
residual.Read(t1, t2);
var fcresid = Reclamation.TimeSeries.Math.HydrometResidualForecast(forecast, qu, residual);
fcresid.WriteToConsole();
}
/// <summary>
/// Lookup hydromet Series.
/// name is interval_cbtt_pcode
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public override ParserResult Lookup(string name, TimeInterval defaultInterval)
{
var interval = defaultInterval;
TimeSeriesName tn = new TimeSeriesName(name);
if (tn.HasInterval)
{
interval = tn.GetTimeInterval();
}
if (tn.Valid)
{
Logger.WriteLine("Hydromet Lookup " + tn.siteid + "," + tn.pcode);
var s = new Series();
if (interval == TimeInterval.Monthly)
{
s = new HydrometMonthlySeries(tn.siteid, tn.pcode, svr);
}
else if (interval == TimeInterval.Irregular)
{
s = new HydrometInstantSeries(tn.siteid, tn.pcode, svr);
}
else if (interval == TimeInterval.Daily)
{
s = new HydrometDailySeries(tn.siteid, tn.pcode, svr);
}
return(new ParserResult(s));
}
else
{
return(base.Lookup(name, interval));
}
}
public static Series Read(string cbtt, string pcode, DateTime t1, DateTime t2,
TimeInterval interval, HydrometHost hydrometServer)
{
Series s = new Series();
if (interval == TimeInterval.Daily)
{
s = HydrometDailySeries.Read(cbtt, pcode, t1, t2, hydrometServer);
}
else if (interval == TimeInterval.Irregular)
{
s = HydrometInstantSeries.Read(cbtt, pcode, t1, t2, hydrometServer);
}
else if (interval == TimeInterval.Monthly)
{
s = new HydrometMonthlySeries(cbtt, pcode,hydrometServer);
s.Read(t1, t2);
}
else
{
throw new ArgumentException("Undefined TimeInterval", interval.ToString());
}
return s;
}
static void Main(string[] args)
{
if (args.Length != 3 && args.Length != 4)
{
Console.WriteLine("Usage: ImportMonthlyValues server user pass [siteid]");
Console.WriteLine(" server = hostname ");
Console.WriteLine(" user = username");
Console.WriteLine(" pass = passwordfile");
return;
}
var host = args[0];
string user = args[1];
string pass = File.ReadAllLines(args[2])[0];
string cbtt = "";
if (args.Length == 4)
{
cbtt = args[3];
}
var svr = PostgreSQL.GetPostgresServer("timeseries", host, user, pass);
//UpdateVMS_daily_por(svr);
TimeSeriesDatabase db = new TimeSeriesDatabase(svr);
Console.WriteLine(db.Server.ConnectionString);
var sitecatalog = db.GetSiteCatalog();
// cbtt, pcode, years, descr, units
DataTable mpoll = new DataTable("mpoll");
if (File.Exists(@"c:\temp\mpoll.xml"))
{
mpoll.ReadXml(@"c:\temp\mpoll.xml");
}
else
{
mpoll = HydrometInfoUtility.MonthlyInventory;
mpoll.WriteXml(@"C:\temp\mpoll.xml", System.Data.XmlWriteMode.WriteSchema);
}
for (int i = 0; i < mpoll.Rows.Count; i++)
{
var r = mpoll.Rows[i];
var siteid = r["cbtt"].ToString().ToLower();
var pcode = r["pcode"].ToString().ToLower();
var years = r["years"].ToString();
if (cbtt != "" && cbtt.ToLower() != siteid.ToLower())
{
continue;
}
HydrometMonthlySeries m = new HydrometMonthlySeries(siteid, pcode, HydrometHost.PN);
HydrometMonthlySeries.ConvertToAcreFeet = false;
// does site id exist in sitecatalog?
if (KeepThisSeries(sitecatalog, siteid, pcode, years))
{
m.Read();
m.RemoveMissing();
Console.WriteLine(siteid + "_" + pcode + " [" + m.Count + "]");
if (m.Count == 0)
{
continue;
}
var folder = db.GetOrCreateFolder("hydromet", siteid, "monthly");
var tn = "monthly_" + siteid + "_" + pcode;
var s = db.GetSeriesFromTableName(tn);
if (s == null)
{ // need to create series.
s = new Series("", TimeInterval.Monthly);
s.Name = siteid + "_" + pcode;
s.Table.TableName = tn;
s.Parameter = pcode;
s.SiteID = siteid;
s.TimeInterval = TimeInterval.Monthly;
db.AddSeries(s, folder);
}
s.Table = m.Table;
db.SaveTimeSeriesTable(s.ID, s, DatabaseSaveOptions.DeleteAllExisting);
}
}
}
private static void SaveTableToSeries(TimeSeriesDatabase db, DataTable table, TimeInterval interval)
{
int i = 1;
string tablePrefix = "daily";
if (interval == TimeInterval.Irregular)
{
tablePrefix = "instant";
}
if (interval == TimeInterval.Monthly)
{
tablePrefix = "monthly";
}
while (i < table.Columns.Count)
{
string tn = table.Columns[i].ColumnName.Trim().ToLower();
tn = tn.Replace(" ", "_");
TimeSeriesName tsn = new TimeSeriesName(tn, interval.ToString().ToLower());
var series1 = db.GetSeriesFromTableName(tn, tablePrefix);
Console.Write(tn + " ");
for (int r = 0; r < table.Rows.Count; r++)
{
var row = table.Rows[r];
object o = row[i];
double val = Point.MissingValueFlag;
if (o != DBNull.Value)
{
val = Convert.ToDouble(row[i]);
}
else
{
continue; // mixing 5 and 15-minute data can cause gaps
}
string flag = "hmet-import";
if (interval == TimeInterval.Irregular || interval == TimeInterval.Monthly)
{
flag = row[i + 1].ToString();
}
DateTime t = Convert.ToDateTime(row[0]);
if (interval == TimeInterval.Monthly)
{
if (tsn.pcode.ToLower() == "fc" || tsn.pcode.ToLower() == "se" || tsn.pcode.ToLower() == "fcm")
{
t = t.FirstOfMonth();
}
if (val != Point.MissingValueFlag && HydrometMonthlySeries.LookupUnits(tsn.pcode) == "1000 acre-feet")
{
val = val * 1000;
}
}
var pt = new Point(t, val, flag);
series1.Add(pt);
}
if (interval == TimeInterval.Irregular || interval == TimeInterval.Monthly)
{
i += 2;// flag column
}
else
{
i++;
}
int rc = series1.Count;
if (rc > 0)
{
rc = db.SaveTimeSeriesTable(series1.ID, series1, DatabaseSaveOptions.UpdateExisting);
}
Console.WriteLine(rc + " records saved");
}
}
private void ProcessFile(string fileName)
{
Logger.WriteLine("ProcessFile('" + fileName + "')");
string importTag = "import"; // used to make friendly export filename
try
{
TextFile tf = new TextFile(fileName);
SeriesList sl = new SeriesList();
if (HydrometInstantSeries.IsValidDMS3(tf))
{
importTag = "decodes";
sl = HydrometInstantSeries.HydrometDMS3DataToSeriesList(tf);
}
else if (HydrometDailySeries.IsValidArchiveFile(tf))
{
importTag = "htools";
sl = HydrometDailySeries.HydrometDailyDataToSeriesList(tf);
}
else if (LoggerNetFile.IsValidFile(tf))
{
LoggerNetFile lf = new LoggerNetFile(tf);
if (lf.IsValid && Array.IndexOf(validSites, lf.SiteName) >= 0)
{
importTag = lf.SiteName;
sl = lf.ToSeries(validPcodes);
}
}
else if (CsvScadaFile.IsValidFile(tf))
{
importTag = "scada";
m_db.GetSiteCatalog();
var scada = new CsvScadaFile(fileName);
sl = scada.ToSeries();
}
else if (XConnectTextFile.IsValidFile(tf))
{
importTag = "xc";
var xc = new XConnectTextFile(tf);
sl = xc.ToSeries();
}
else if (HydrometMonthlySeries.IsValidFile(tf))
{
sl = HydrometMonthlySeries.FileToSeriesList(tf);
}
else
{
Logger.WriteLine("skipped Unknown File Format: " + fileName);
return;
}
if (m_importer == null)
{
m_importer = new TimeSeriesImporter(m_db, m_saveOption);
}
Console.WriteLine("Found " + sl.Count + " series in " + fileName);
foreach (var item in sl)
{
Logger.WriteLine(item.Table.TableName);
}
if (sl.Count > 0)
{
m_importer.Import(sl, m_computeDependencies, m_computeDailyDependencies, importTag);
FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "attic", fileName);
}
}
catch (Exception ex)
{
Logger.WriteLine("Error:" + ex.Message);
Console.WriteLine("Error: skipping file, will move to error subdirectory " + fileName);
FileUtility.MoveToSubDirectory(Path.GetDirectoryName(fileName), "error", fileName);
}
}
