/// <summary>
/// Find all CalculationSeries (Equations) that reference this Series. Use the tablename
/// of this series as the variable to look for in each equation.
/// </summary>
/// <param name="cs"></param>
/// <returns></returns>
private List <CalculationSeries> BuildDependencies(CalculationSeries cs)
{
var rval = new List <CalculationSeries>();
var vars = cs.GetDependentVariables();
string msg = cs.Table.TableName + " depends on :";
TimeSeriesName cName = new TimeSeriesName(cs.Table.TableName);
foreach (var vn in vars)
{
TimeSeriesName tn = new TimeSeriesName(vn, cName.interval);
if (tn.GetTableName() == cs.Table.TableName)
{
Logger.WriteLine(cs.Expression);
Logger.WriteLine("Warning: prevented recursive dependency " + tn.GetTableName());
continue;
}
var dependents = list.FindAll(x => x.Table.TableName == tn.GetTableName());
foreach (var d in dependents)
{
msg += d.Table.TableName + ",";
}
rval.AddRange(dependents);
}
Logger.WriteLine(msg);
return(rval);
}
public int AddRow(string siteID, int parentid, string units, string pcode, string expression = "", TimeInterval interval = TimeInterval.Irregular)
{
var provider = "Series";
string iconName = "";
if (expression != "")
{
provider = "CalculationSeries";
iconName = "sum";
}
var tn = new TimeSeriesName(siteID + "_" + pcode, interval);
string tableName = tn.GetTableName();
var rows = Select("tablename = '" + tableName + "'");
if (rows.Length > 0)
{
Console.WriteLine("Warning table:'" + tableName + "' allready exists");
}
int rval = NextID();
AddSeriesCatalogRow(rval, parentid, 0, 1, iconName, siteID + "_" + pcode, siteID, units, interval.ToString(),
pcode, tableName, provider, "", expression, "", 1);
return(rval);
}
private void CreateMultiColumnSchema()
{
if (m_seriesList.Count == 0)
{
return;
}
var tbl = m_seriesList[0].Table;
AppendColumn(tbl.Columns[0], tbl.Columns[0].ColumnName); // DateTime
m_columnToSeries.Add(0);
for (int i = 0; i < m_seriesList.Count; i++)
{
var s = m_seriesList[i];
tbl = s.Table;
TimeSeriesName tn = new TimeSeriesName(s.Table.TableName);
AppendColumn(tbl.Columns[1], tn.siteid.PadRight(8) + tn.pcode); // value column
m_columnToSeries.Add(i);
if (s.HasFlags)
{
AppendColumn(tbl.Columns[2], "flag" + (i + 1)); // flag
m_columnToSeries.Add(i);
}
}
}
/// <summary>
/// Finds a matching row
/// TableMask can be exact or with wildcards.
/// For example *_ob will apply to all table names that end with ‘_ob’.
/// Flags will not be applied to tables that are not defined by a mask.
/// Also nulls in the columns (High, Low, Change) will cause no flagging to occur.
/// </summary>
/// <param name="tableName"></param>
/// <returns></returns>
public TimeSeriesDatabaseDataSet.quality_limitRow GetRow(string tableName)
{
// try exact match first. instant_odsw_ob
var rows = m_limit.Select("TableMask = '" + tableName + "'");
if (rows.Length == 1)
{
return(rows[0] as TimeSeriesDatabaseDataSet.quality_limitRow);
}
TimeSeriesName tn = new TimeSeriesName(tableName);
// try site specific next: odsw_ob
var mask = tn.siteid + "_" + tn.pcode;
rows = m_limit.Select("TableMask = '" + mask + "'");
if (rows.Length == 1)
{
return(rows[0] as TimeSeriesDatabaseDataSet.quality_limitRow);
}
// try parameter alone next
mask = tn.pcode;
rows = m_limit.Select("TableMask like '[*]_" + mask + "'");
if (rows.Length == 1)
{
return(rows[0] as TimeSeriesDatabaseDataSet.quality_limitRow);
}
return(null);
}
/// <summary>
/// gets daily dependents for this series (tablename)
/// </summary>
/// <param name="tableName"></param>
/// <param name="t"></param>
/// <returns></returns>
private List <CalculationSeries> GetDailyDependents(string tableName)
{
var rval = new List <CalculationSeries>();
TimeSeriesName tn = new TimeSeriesName(tableName);
var calcList = GetDependentCalculations(tableName, TimeInterval.Daily);
//if (calcList.Count > 0)
// Logger.WriteLine("Found " + calcList.Count + " daily calculations to update ref:"+tableName);
foreach (var item in calcList)
{
if (!rval.Any(a => a.Table.TableName == item.Table.TableName))
{
rval.Add(item);
}
// check for daily that depends on daily.
var x = GetDailyDependents(item.Table.TableName);
foreach (var d in x)
{
if (!rval.Any(a => a.Table.TableName == d.Table.TableName))
{
rval.Add(d);
}
}
}
return(rval);
}
/// <summary>
/// Export data to outgoing directory
/// </summary>
/// <param name="importTag"></param>
/// <param name="routingList"></param>
private void Export(string importTag, SeriesList routingList)
{
SeriesList instantRoute = new SeriesList();
SeriesList dailyRoute = new SeriesList();
// route data to other locations.
foreach (var item in routingList)
{
TimeSeriesName tn = new TimeSeriesName(item.Table.TableName);
item.Parameter = tn.pcode;
item.SiteID = tn.siteid;
if (item.TimeInterval == TimeInterval.Irregular)
{
instantRoute.Add(item);
}
if (item.TimeInterval == TimeInterval.Daily)
{
dailyRoute.Add(item);
}
}
Logger.WriteLine("Exporting data");
TimeSeriesTransfer export = new TimeSeriesTransfer(m_db);
export.Export(instantRoute, importTag, TimeInterval.Irregular);
export.Export(dailyRoute, importTag, TimeInterval.Daily);
}
/// <summary>
/// read computed value from hydromet and compare..
///
/// </summary>
/// <param name="s"></param>
private void CompareToHydromet(CalculationSeries s)
{
if (s.Count == 0)
{
return;
}
// TO DO.. also check instant calcs.
TimeSeriesName n = new TimeSeriesName(s.Table.TableName);
if (s.TimeInterval == TimeInterval.Daily)
{
var tmp = HydrometDailySeries.Cache;
HydrometDailySeries.Cache = null; // don't use cache..
HydrometDailySeries h = new HydrometDailySeries(n.siteid, n.pcode);
HydrometDailySeries.Cache = tmp;
h.Read(s.MinDateTime, s.MaxDateTime);
Series diff = Reclamation.TimeSeries.Math.Abs(h - s);
var pt = Reclamation.TimeSeries.Math.MaxPoint(diff);
double tolerance = 0.1;
if (Array.IndexOf(new string[] { "sr", "wr" }, n.pcode) >= 0)
{
tolerance = 1.0;
}
double delta = h[0].Value - s[0].Value;
double pctError = 0;
if (System.Math.Abs(delta) > 0)
{
pctError = System.Math.Abs(delta / h[0].Value);
}
if (pctError > tolerance)
{
if (debugFileName == "")
{
debugFileName = "calc_errors.csv";
if (File.Exists(debugFileName))
{
File.Delete(debugFileName);
}
File.AppendAllText(debugFileName, "site,pcode,interval,openvms,linux,delta,percentError" + "\n");
}
if (h[0].Value != Point.MissingValueFlag)
{
string msg = n.siteid + "," + n.pcode + "," + n.interval + "," + h[0].Value + ", " + s[0].Value + ", " + delta + ", " + pctError;
Console.WriteLine(msg);
File.AppendAllText(debugFileName, msg + "\n");
}
}
}
else
{
throw new NotImplementedException();
}
}
static bool s_appendToFile = false; // for output file.
private static void WriteToHydrometDailyFile(string dailyFileName, CalculationSeries s)
{
TimeSeriesName n = new TimeSeriesName(s.Table.TableName);
HydrometDailySeries.WriteToArcImportFile(s, n.siteid, n.pcode,
dailyFileName, s_appendToFile);
if (!s_appendToFile)
{
s_appendToFile = true; // append after the first time.
}
}
public CalculationSeries[] GetDependentCalculations(string siteID, string pcode)
{
TimeSeriesDependency td = new TimeSeriesDependency(m_dependencyList);
TimeSeriesName tn = new TimeSeriesName(siteID + "_" + pcode, m_interval);
var list = td.LookupCalculations(tn.GetTableName(), m_interval).ToArray();
var cList = new List <CalculationSeries>();
foreach (var item in list)
{
if (item is CalculationSeries)
{
cList.Add(item as CalculationSeries);
}
}
return(cList.ToArray());
}
public static void Set(string name, string value, TimeSeriesName tn, BasicDBServer svr)
{
var tableName = tn.GetTableName();
var sc = svr.Table("seriescatalog", "select * from seriescatalog where tablename = '" + tableName + "'");
if (sc.Rows.Count == 1)
{
int id = Convert.ToInt32(sc.Rows[0]["id"]);
Set(name, value, id, svr);
}
else
{
var msg = "Error: tablename:" + tableName + "not found (or duplicated) in the seriescatalog";
Logger.WriteLine(msg);
throw new KeyNotFoundException(msg);
}
}
/// <summary>
/// gets daily dependents for this series (tablename)
/// </summary>
/// <param name="tableName"></param>
/// <param name="t"></param>
/// <returns></returns>
private SeriesList GetDailyDependents(string tableName)
{
SeriesList rval = new SeriesList();
TimeSeriesName tn = new TimeSeriesName(tableName);
var calcList = GetDependentCalculations(tableName, TimeInterval.Daily);
//if (calcList.Count > 0)
// Logger.WriteLine("Found " + calcList.Count + " daily calculations to update ref:"+tableName);
foreach (var item in calcList)
{
// prevent recursive?
rval.Add(item);
// check for daily that depends on daily.
var x = GetDailyDependents(item.Table.TableName);
rval.AddRange(x);
}
return(rval);
}
/// <summary>
/// gets dependents for this series (tablename)
/// </summary>
/// <param name="tableName"></param>
/// <param name="t"></param>
/// <returns></returns>
private List <CalculationSeries> GetDependentsRecursive(string tableName, TimeInterval interval, ref int level)
{
string pad = "".PadRight(level);
Logger.WriteLine(pad + "GetDependentsRecursive(" + tableName + "," + interval.ToString() + ",depth=" + level + ")");
int maxRecursiveCount = 20;
if (level > maxRecursiveCount)
{
Logger.WriteLine(pad + "TimeSeriesImporter max recursive level reached. " + tableName + " maxRecursiveCount =" + maxRecursiveCount);
//level = 0;
return(new List <CalculationSeries>());
}
var rval = new List <CalculationSeries>();
TimeSeriesName tn = new TimeSeriesName(tableName);
var calcList = GetDependentCalculations(tableName, interval);
foreach (CalculationSeries c in calcList)
{
if (!rval.Any(a => a.Table.TableName == c.Table.TableName))
{
rval.Add(c);
Logger.WriteLine(pad + " " + c.Name + " =" + c.Expression);
}
// check recursive
if (tableName != c.Table.TableName)
{
level++;
pad = "".PadRight(level);
var x = GetDependentsRecursive(c.Table.TableName, interval, ref level);
foreach (var d in x)
{
if (!rval.Any(a => a.Table.TableName == d.Table.TableName))
{
rval.Add(d);
Logger.WriteLine(pad + " -- " + d.Name + " = " + d.Expression);
}
}
}
}
return(rval);
}
///// <summary>
///// determines if the child is inside the root
///// </summary>
///// <param name="selectedPath">root path </param>
///// <param name="rowPath"></param>
///// <returns></returns>
// private bool IsChildInRoot(List<string> root, List<string> child)
// {
// if (root.Count > child.Count)
// return false;
// for (int i = 0; i < root.Count; i++)
// {
// if (root[i] != child[i])
// return false;
// }
// return true;
// }
private static string GetIntervalPath(TimeSeriesDatabaseDataSet.SeriesCatalogRow row)
{
TimeSeriesName tn = new TimeSeriesName(row.TableName);
var interval = "instant";
if (row.TimeInterval == "Irregular" || row.TimeInterval == "Hourly")
{
interval = "instant";
}
else
{
interval = row.TimeInterval.ToLower();
}
if (TimeSeriesDatabase.IsQuality(tn.pcode))
{
interval = "quality";
}
return(interval);
}
/// <summary>
/// Return list of all calculations that may need to be performed
/// based on this inputSeries
/// </summary>
/// <param name="inputSeries"></param>
/// <returns></returns>
public SeriesList LookupCalculations(string tableName, TimeInterval interval)
{
var rval = new SeriesList();
if (inputDictionary == null)
{
inputDictionary = new Dictionary <string, List <CalculationSeries> >();
foreach (CalculationSeries cs in list)
{
var vars = cs.GetDependentVariables();
foreach (var varName in vars)
{
TimeSeriesName tn = new TimeSeriesName(varName, interval);
if (!tn.Valid)
{
Console.WriteLine("Error: Skipped Invalid equation .... " + cs.Expression);
}
else
{
AddToDictionary(tn.GetTableName(), cs);
}
}
}
}
Logger.WriteLine("LookupCalculations(" + tableName + ")");
Logger.WriteLine("inputDictionary.Count = " + inputDictionary.Count);
TimeSeriesName n = new TimeSeriesName(tableName, interval);
var key = n.GetTableName();
if (this.inputDictionary.ContainsKey(key))
{
rval.AddRange(inputDictionary[key].ToArray());
}
return(rval);
}
/// <summary>
/// Return list of all calculations that may need to be performed
/// based on this inputSeries
/// </summary>
/// <param name="inputSeries"></param>
/// <returns></returns>
public List <CalculationSeries> LookupCalculations(string tableName, TimeInterval interval)
{
var rval = new List <CalculationSeries>();
InitInputDictionary(interval);
TimeSeriesName n = new TimeSeriesName(tableName, interval);
var key = n.GetTableName();
if (this.inputDictionary.ContainsKey(key))
{
//Logger.WriteLine("inputDictionary.Count = " + inputDictionary.Count);
//Logger.WriteLine("LookupCalculations(" + tableName + ")");
var x = inputDictionary[key];
rval.AddRange(x.ToArray());
}
return(rval);
}
private void InitInputDictionary(TimeInterval interval)
{
if (inputDictionary == null)
{
inputDictionary = new Dictionary <string, List <CalculationSeries> >();
foreach (CalculationSeries cs in list)
{
var vars = cs.GetDependentVariables();
foreach (var varName in vars)
{
TimeSeriesName tn = new TimeSeriesName(varName, interval);
if (!tn.Valid)
{
Console.WriteLine("Warning: Skipped non-series parameter/variable .... " + varName);
}
else
{
AddToDictionary(tn.GetTableName(), cs);
}
}
}
}
}
/// <summary>
/// Calculates a group of daily values.
/// </summary>
/// <param name="t1"></param>
/// <param name="t2"></param>
/// <param name="propertyFilter">series property filter. Example program:agrimet </param>
/// <param name="simulate">simulate calculations, don't actuually do it.</param>
public CalculationSeries[] ComputeDailyValues(DateTime t1, DateTime t2, bool compareToHydromet = false, string errorFileName = "",
string detailFileName = "", bool simulate = false)
{
Performance p = new Performance();
HydrometInstantSeries.Cache = new HydrometDataCache(); // clear out and make new cache.
string dailyFileName = GetDailyFileName(m_propertyFilter);
bool appendToFile = false; // for output file.
Console.WriteLine("Computing daily values for " + m_dependencyList.Count + " series");
TimeSeriesDependency td = new TimeSeriesDependency(m_dependencyList);
var sorted = td.Sort();
foreach (var s in sorted)
{
if (!s.Enabled)
{
continue; // calculations turned off
}
string originalExpression = s.Expression;
// compute Values
if (m_db.Parser.VariableResolver is HydrometVariableResolver)
{
CacheAllParametersForSite(s, t1, t2); // 50% performance boost.
}
Console.Write(s.Table.TableName + " = " + s.Expression);
if (simulate)
{
Console.WriteLine("skipping calc");
continue;
}
s.Calculate(t1, t2); // Calculate() also saves to local time series database.
if (s.Count == 0 || s.CountMissing() > 0)
{
File.AppendAllText(errorFileName, "Error: " + s.Table.TableName + " = " + s.Expression + "\n");
string msg = "\nDetails: " + s.Table.TableName + " = " + s.Expression + "\n";
foreach (var x in s.Messages)
{
msg += "\n" + x;
}
Console.WriteLine(msg);
File.AppendAllText(detailFileName, msg);
}
else
{
// File.AppendAllText(errorFileName, " OK. ");
Console.WriteLine(" OK. ");
}
if (compareToHydromet)
{
CompareToHydromet(s);
}
s.Expression = originalExpression;
TimeSeriesName n = new TimeSeriesName(s.Table.TableName);
HydrometDailySeries.WriteToArcImportFile(s, n.siteid, n.pcode, dailyFileName, appendToFile);
if (!appendToFile)
{
appendToFile = true; // append after the first time.
}
}
if (appendToFile) // might not have any results
{
Console.WriteLine("Results Saved to " + dailyFileName);
}
p.Report(); // 185 seconds
return(sorted);
}
/// <summary>
/// Imports time series data,
/// 1) set flags
/// 2) active alarms (TO DO)
/// 3) compute dependent data (same interval)
/// 4) compute daily data when encountering midnight values
/// </summary>
/// <param name="inputSeriesList"></param>
/// <param name="computeDependencies"></param>
/// <param name="computeDailyEachMidnight"></param>
public void Import(SeriesList inputSeriesList,
bool computeDependencies = false,
bool computeDailyEachMidnight = false,
string importTag = "data")
{
var calculationQueue = new SeriesList();
var routingList = new SeriesList();
foreach (var s in inputSeriesList)
{
// set flags.
Logger.WriteLine("Checking Flags ");
m_db.Quality.SetFlags(s); // to do, log/email flagged data
// To Do.. check for alarms..
m_db.ImportSeriesUsingTableName(s, "");
routingList.Add(s);
if (computeDependencies)
{
var z = ComputeDependenciesSameInterval(s);
routingList.AddRange(z);
}
if (computeDailyEachMidnight)
{
var x = GetDailyCalculationsIfMidnight(s);
foreach (var item in x)
{
if (!calculationQueue.ContainsTableName(item))
{
calculationQueue.Add(item);
}
}
}
}
if (calculationQueue.Count > 0)
{
// do Actual Computations now. (in proper order...)
var list = new List <CalculationSeries>();
foreach (Series item in calculationQueue)
{
list.Add(item as CalculationSeries);
}
TimeSeriesDependency td = new TimeSeriesDependency(list);
var sortedCalculations = td.Sort();
foreach (CalculationSeries cs in sortedCalculations)
{
Console.Write(">>> " + cs.Table.TableName + ": " + cs.Expression);
//var cs = item as CalculationSeries;
var t1 = inputSeriesList.MinDateTime.Date;
var t2 = inputSeriesList.MaxDateTime;
if (t1.Date == t2.AddDays(-1).Date) // spans midnight, compute yesterday.
{
t1 = t1.Date;
t2 = t1.Date;
}
cs.Calculate(t1, t2);
if (cs.Count > 0)
{
routingList.Add(cs);
if (cs.CountMissing() > 0)
{
Console.WriteLine(" Missing " + cs.CountMissing() + " records");
}
else
{
Console.WriteLine(" OK");
}
}
}
}
SeriesList instantRoute = new SeriesList();
SeriesList dailyRoute = new SeriesList();
// route data to other locations.
foreach (var item in routingList)
{
TimeSeriesName tn = new TimeSeriesName(item.Table.TableName);
item.Parameter = tn.pcode;
item.SiteID = tn.siteid;
if (item.TimeInterval == TimeInterval.Irregular)
{
instantRoute.Add(item);
}
if (item.TimeInterval == TimeInterval.Daily)
{
dailyRoute.Add(item);
}
}
Console.WriteLine("Routing data");
TimeSeriesRouting.RouteInstant(instantRoute, importTag, m_routing);
TimeSeriesRouting.RouteDaily(dailyRoute, importTag, m_routing);
}
