private void WriteListToFile(SeriesList list, string name, TimeInterval interval, string route)
{
var tmpFileName = FileUtility.GetTempFileName(".txt");
File.Delete(tmpFileName);
Logger.WriteLine("writing " + list.Count + " series ");
Logger.WriteLine("temp file:" + tmpFileName);
if (interval == TimeInterval.Daily)
{
SeriesList exportList = new SeriesList();
foreach (var s in list)
{
exportList.Add(s);
}
HydrometDailySeries.WriteToArcImportFile(exportList, tmpFileName);
}
if (interval == TimeInterval.Irregular)
{
foreach (var s in list)
{
HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, Environment.UserName, tmpFileName, true);
}
}
if (File.Exists(tmpFileName))
{
Logger.WriteLine("Moving: " + tmpFileName);
string fn = interval == TimeInterval.Daily ? "daily" : "instant";
var fileName = GetOutgoingFileName(fn, name, "all", route);
Logger.WriteLine("To: " + fileName);
File.Move(tmpFileName, fileName);
}
}
public void Add(SeriesList items)
{
foreach (Series s in items)
{
Add(s);
}
}
/// <summary>
/// Imports list of series
/// </summary>
/// <param name="importSeries"></param>
/// <param name="computeDependencies"></param>
/// <param name="importTag"></param>
/// <returns>list of series including dependent calculations</returns>
private SeriesList Import(SeriesList importSeries, bool computeDependencies, string importTag)
{
Performance p = new Performance();
int calculationCount = 0;
var routingList = new SeriesList();
foreach (var s in importSeries)
{
m_db.Quality.SetFlags(s); // to do, log/email flaged data
m_db.ImportSeriesUsingTableName(s, m_saveOption);
CheckForAlarms(s);
routingList.Add(s);
if (computeDependencies)
{
var z = ComputeDependenciesSameInterval(s);
calculationCount += z.Count;
foreach (var cs in z)
{
CheckForAlarms(cs);
}
routingList.AddRange(z);
}
}
Export(importTag, routingList);
Stats(importSeries.Count(), p.ElapsedSeconds, calculationCount, "instant");
return(routingList);
}
private SeriesList ComputeDependenciesSameInterval(Series s)
{
int level = 0;
SeriesList rval = new SeriesList();
var calcList = GetDependentsRecursive(s.Table.TableName, s.TimeInterval, ref level);
if (calcList.Count > 0)
{
Logger.WriteLine("Found " + calcList.Count + " " + s.TimeInterval + " calculations to update ");
}
TimeSeriesDependency d = new TimeSeriesDependency(calcList);
var sorted = d.Sort();
foreach (var item in sorted)
{
var cs = item as CalculationSeries;
cs.Calculate(s.MinDateTime, s.MaxDateTime);
if (cs.Count > 0)
{
rval.Add(cs);
}
}
return(rval);
}
/// <summary>
/// Routes a list of Series as a group
/// hydromet cbtt is copied from list[i].SiteName
/// hydromet pcode is copied from list[i].Parameter
/// </summary>
/// <param name="list"></param>
/// <param name="route"></param>
/// <param name="name">identity used as part of filename </param>
public static void RouteDaily(SeriesList list, string name, RouteOptions route = RouteOptions.Both)
{
if (list.Count == 0)
{
return;
}
string fileName = "";
if (route == RouteOptions.Both || route == RouteOptions.Outgoing)
{
fileName = GetOutgoingFileName("daily", name, "all");
Console.WriteLine("saving daily outgoing to:" + fileName);
HydrometDailySeries.WriteToArcImportFile(list, fileName);
}
if (route == RouteOptions.Both || route == RouteOptions.Incoming)
{
foreach (var s in list)
{
fileName = GetIncommingFileName("daily", s.SiteID, s.Parameter);
Console.WriteLine("saving daily incoming to:" + fileName);
s.WriteCsv(fileName, true);
}
}
}
private List <CalculationSeries> GetDailyDependentCalculations(SeriesList list)
{
Logger.WriteLine("GetDailyDependentCalculations(" + list.Count + " series )");
var rval = new List <CalculationSeries>();
foreach (var s in list)
{
int level = 0;
if (NeedDailyCalc(s))
{ // daily calcs that depend on instant
var x = GetDependentsRecursive(s.Table.TableName, TimeInterval.Daily, ref level);
if (x.Count > 0)
{
Logger.WriteLine("for " + s.Table.TableName + " found " + x.Count + " dependencies (before checking for duplicates)");
}
foreach (var item in x)
{ // prevent duplicates
if (!rval.Any(a => a.Table.TableName == item.Table.TableName))
{
rval.Add(item);
Logger.WriteLine(" --> " + item.Table.TableName);
}
}
}
}
return(rval);
}
/// <summary>
/// Routes a list of Series as a group
/// hydromet cbtt is copied from list[i].SiteName
/// hydromet pcode is copied from list[i].Parameter
/// </summary>
/// <param name="list"></param>
/// <param name="route"></param>
/// <param name="name">identity used as part of filename </param>
public static void RouteInstant(SeriesList list, string name, RouteOptions route = RouteOptions.Both)
{
if (route == RouteOptions.None)
{
return;
}
if (route == RouteOptions.Both || route == RouteOptions.Outgoing)
{
var tmpFileName = FileUtility.GetTempFileName(".txt");
File.Delete(tmpFileName);
Console.WriteLine("temp file:" + tmpFileName);
foreach (var s in list)
{
HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, WindowsUtility.GetShortUserName(), tmpFileName, true);
}
Console.WriteLine("Moving: " + tmpFileName);
var fileName = GetOutgoingFileName("instant", name, "all");
Console.WriteLine("To: " + fileName);
File.Move(tmpFileName, fileName);
}
else
{
throw new NotImplementedException("incoming not supported.");
}
}
public Series Sum(DateTime t1, DateTime t2)
{
SeriesList sl = new SeriesList();
sl = this.Subset(t1, t2);
return(sl.Sum());
}
/// <summary>
/// Imports time series data,
/// 1) set flags
/// 2) active alarms (TO DO)
/// 3) compute dependent data (same interval)
/// 4) compute daily data when encountering midnight values
/// </summary>
/// <param name="importSeries"></param>
/// <param name="computeDependencies"></param>
/// <param name="computeDailyDependencies"></param>
public void Import(SeriesList importSeries,
bool computeDependencies = false,
bool computeDailyDependencies = false,
string importTag = "data")
{
var calculationQueue = new List <CalculationSeries>();
var routingList = new SeriesList();
foreach (var s in importSeries)
{
m_db.Quality.SetFlags(s); // to do, log/email flaged data
ProcessAlarms(s);
var folderNames = SetupDefaultFolders(s);
m_db.ImportSeriesUsingTableName(s, folderNames, m_saveOption);
routingList.Add(s);
if (computeDependencies)
{
var z = ComputeDependenciesSameInterval(s);
routingList.AddRange(z);
}
if (computeDailyDependencies && NeedDailyCalc(s))
{ // daily calcs that depend on instant
GetDailyDependentCalculations(s, calculationQueue);
}
}
if (calculationQueue.Count > 0)
{
PerformDailyComputations(importSeries, calculationQueue, routingList);
}
RouteData(importTag, routingList);
}
private SeriesList GetDailyCalculationsIfMidnight(Series s)
{
var calcList = new SeriesList();
// check for midnight values, and initiate daily calculations.
if (s.TimeInterval == TimeInterval.Irregular)
{
for (int i = 0; i < s.Count; i++)
{
var pt = s[i];
if (pt.DateTime.IsMidnight())
{
var x = GetDailyDependents(s.Table.TableName);
foreach (var item in x)
{
if (!calcList.ContainsTableName(item))
{
calcList.AddRange(x);
}
}
}
}
}
return(calcList);
}
public SimpleMathSeries(TimeSeriesDatabase db, Reclamation.TimeSeries.TimeSeriesDatabaseDataSet.SeriesCatalogRow sr) : base(db, sr)
{
string[] idList = ConnectionStringUtility.GetToken(ConnectionString, "SeriesList", "").Split(',');
string[] opList = ConnectionStringUtility.GetToken(ConnectionString, "OperatorList", "").Split(',');
m_operation = new MathOperation[opList.Length];
m_items = new SeriesList();
for (int i = 0; i < idList.Length; i++)
{
int id = -1;
if (!Int32.TryParse(idList[i], out id) || !db.SeriesExists(id))
{
Logger.WriteLine("Cannot calculate series. Has the data this calculation needs been deleted?");
Name = "Error: source data missing " + Name;
m_sourceDeleted = true;
break;
}
else
{
Series s = db.GetSeries(id);
m_items.Add(s);
}
}
ScenarioName = m_items[0].ScenarioName;
// TO DO assuming all scenarioNames are the same.
for (int i = 0; i < opList.Length; i++)
{
m_operation[i] = MathOperationFromString(opList[i]);
}
}
/// <summary>
/// Imports time series data,
/// 1) set flags
/// 2) active alarms (TO DO)
/// 3) compute dependent data (same interval)
/// 4) compute daily data when encountering midnight values
/// </summary>
/// <param name="importSeries"></param>
/// <param name="computeDependencies"></param>
/// <param name="computeDailyDependencies"></param>
public void Import(SeriesList importSeries,
bool computeDependencies = false,
bool computeDailyDependencies = false,
string importTag = "data")
{
var calculationQueue = new List <CalculationSeries>();
var routingList = new SeriesList();
foreach (var s in importSeries)
{
m_db.Quality.SetFlags(s); // to do, log/email flaged data
m_db.Alarms.Check(s); // check for alarms; send email make phone calls
var folderNames = new string[] {}; // TO DO //hydromet/cbtt default.
m_db.ImportSeriesUsingTableName(s, folderNames, m_saveOption);
routingList.Add(s);
if (computeDependencies)
{
var z = ComputeDependenciesSameInterval(s);
routingList.AddRange(z);
}
if (computeDailyDependencies && NeedDailyCalc(s))
{ // daily calcs that depend on instant
GetDailyDependentCalculations(s, calculationQueue);
}
}
if (calculationQueue.Count > 0)
{
PerformDailyComputations(importSeries, calculationQueue, routingList);
}
RouteData(importTag, routingList);
}
/// <summary>
/// Export data to outgoing directory
/// </summary>
/// <param name="importTag"></param>
/// <param name="routingList"></param>
private void Export(string importTag, SeriesList routingList)
{
SeriesList instantRoute = new SeriesList();
SeriesList dailyRoute = new SeriesList();
// route data to other locations.
foreach (var item in routingList)
{
TimeSeriesName tn = new TimeSeriesName(item.Table.TableName);
item.Parameter = tn.pcode;
item.SiteID = tn.siteid;
if (item.TimeInterval == TimeInterval.Irregular)
{
instantRoute.Add(item);
}
if (item.TimeInterval == TimeInterval.Daily)
{
dailyRoute.Add(item);
}
}
Logger.WriteLine("Exporting data");
TimeSeriesTransfer export = new TimeSeriesTransfer(m_db);
export.Export(instantRoute, importTag, TimeInterval.Irregular);
export.Export(dailyRoute, importTag, TimeInterval.Daily);
}
/// <summary>
/// Imports a single series (used for testing)
/// </summary>
/// <param name="s"></param>
/// <param name="computeDependencies"></param>
/// <param name="computeDailyEachMidnight"></param>
internal void Import(Series s, bool computeDependencies = false,
bool computeDailyEachMidnight = false)
{
var sl = new SeriesList();
sl.Add(s);
Import(sl, computeDependencies, computeDailyEachMidnight, "test");
}
public SeriesList AggregateAndSubset(StatisticalMethods aggregateType, MonthDayRange monthDayRange, int beginningMonth)
{
SeriesList rval = new SeriesList();
foreach (Series s in this)
{
rval.Add(TimeSeries.Math.AggregateAndSubset(aggregateType, s, monthDayRange, beginningMonth));
}
return(rval);
}
public SeriesList Subset(MonthDayRange range)
{
SeriesList rval = new SeriesList();
foreach (Series s in this)
{
rval.Add(TimeSeries.Math.Subset(s, range));
}
return(rval);
}
public SeriesList Subset(int[] months)
{
SeriesList rval = new SeriesList();
foreach (Series s in this)
{
rval.Add(TimeSeries.Math.Subset(s, months));
}
return(rval);
}
List <int> m_columnToSeries; // index to series from each column in table
/// <summary>
/// Creates a DataTable composed from multiple time series
/// </summary>
public SeriesListDataTable(SeriesList list,
TimeInterval interval) : base(interval.ToString())
{
m_seriesList = list.FilteredList(interval);
m_columnToSeries = new List <int>();
CreateMultiColumnSchema();
AddData();
AcceptChanges();
ColumnChanged += new DataColumnChangeEventHandler(m_table_ColumnChanged);
SetupPrimaryKey();
}
/// <summary>
/// Imports time series data,
/// 1) set flags
/// 2) active alarms
/// 3) compute dependent data (same interval)
/// 4) compute daily data when encountering midnight values
/// </summary>
/// <param name="importSeries"></param>
/// <param name="computeDependencies"></param>
/// <param name="computeDailyDependencies"></param>
public void Import(SeriesList importSeries,
bool computeDependencies = false,
bool computeDailyDependencies = false,
string importTag = "data")
{
var list = Import(importSeries, computeDependencies, importTag);
if (computeDailyDependencies)
{
DailyCalculations(list, importTag);
}
}
public SeriesList SummaryHydrograph(int[] exceedanceLevels, DateTime beginningDate, bool max, bool min, bool avg, bool removeEmptySeries)//, bool removeFeb29)
{
SeriesList rval = new SeriesList();
foreach (Series s in this)
{
SeriesList list = Math.SummaryHydrograph(s, exceedanceLevels, beginningDate, max, min, avg, removeEmptySeries);//,removeFeb29);
rval.AddRange(list);
}
return(rval);
}
public static Series MLRInterpolationPisces(double fitTolerance, params Series[] s)
{
SeriesList sList = new SeriesList();
foreach (var item in s)
{
sList.Add(item);
}
var sOut = Reclamation.TimeSeries.Estimation.MultipleLinearRegression.MlrInterpolation(sList,
new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, fitTolerance);
return(sOut.EstimatedSeries);
}
///// <summary>
///// Subtracts Series within a list from series in another list
///// </summary>
///// <param name="a"></param>
///// <param name="b"></param>
///// <returns></returns>
//public static SeriesList operator -(SeriesList a, SeriesList b)
//{
// if (a.Count != b.Count)
// throw new ArgumentException("Error: number of series is not the same in the two lists");
// var rval = new SeriesList();
// for (int i = 0; i < a.Count; i++)
// {
// var sa = a[i];
// var sb = b[i];
// rval.Add(Math.Subtract(sa, sb));
// }
// return rval;
//}
///// <summary>
///// Adds Series within a list to series in another list
///// </summary>
///// <param name="a"></param>
///// <param name="b"></param>
///// <returns></returns>
//public static SeriesList operator +(SeriesList a, SeriesList b)
//{
// if (a.Count != b.Count)
// throw new ArgumentException("Error: number of series is not the same in the two lists");
// var rval = new SeriesList();
// for (int i = 0; i < a.Count; i++)
// {
// var sa = a[i];
// var sb = b[i];
// rval.Add(Math.Add(sa, sb));
// }
// return rval;
//}
/// <summary>
/// Creates a new SeriesList Filtered by interval
/// </summary>
/// <param name="interval"></param>
/// <returns></returns>
public SeriesList FilteredList(TimeInterval interval)
{
var rval = new SeriesList();
foreach (var item in this)
{
if (item.TimeInterval == interval)
{
rval.Add(item);
}
}
return(rval);
}
public SeriesList Subset(DateTime d1, DateTime d2)
{
string datecol = this[0].Table.Columns[0].ColumnName;
SeriesList rval = new SeriesList();
String sql = "[" + datecol + "] >= '" + d1.ToShortDateString() +
"' AND [" + datecol + "] <= '" + d2.ToShortDateString() + "'";
foreach (Series s in this)
{
Series ss = s.Subset(sql);
rval.Add(ss);
}
return(rval);
}
/// <summary>
/// Routes a list of Series as a group
/// hydromet cbtt is copied from list[i].SiteName
/// hydromet pcode is copied from list[i].Parameter
/// </summary>
/// <param name="list"></param>
/// <param name="route"></param>
/// <param name="name">identity used as part of filename </param>
public void Export(SeriesList list, string name, TimeInterval interval)
{
if (list.Count == 0)
{
return;
}
var routes = RouteDestinations(list);
foreach (string route in routes.Keys)
{
WriteListToFile(routes[route], name, interval, route);
}
}
private void DailyCalculations(SeriesList importSeries, string importTag)
{
Logger.WriteLine("DailyCalculations(" + importSeries.Count + " series, " + importTag + ")");
Performance p = new Performance();
var dailyCalculationQueue = GetDailyDependentCalculations(importSeries);
var routingList = new SeriesList();
if (dailyCalculationQueue.Count > 0)
{
PerformDailyComputations(importSeries, dailyCalculationQueue, routingList);
}
Export(importTag, routingList);
Stats(0, p.ElapsedSeconds, dailyCalculationQueue.Count, "daily");
}
private static void AddToRoute(Dictionary <string, SeriesList> rval, Series s, string key)
{
SeriesList sl = null;
if (!rval.ContainsKey(key))
{
sl = new SeriesList();
rval.Add(key, sl);
}
else
{
sl = rval[key];
}
sl.Add(s);
}
public override Series CreateScenario(TimeSeriesDatabaseDataSet.ScenarioRow scenario)
{
SeriesList list = new SeriesList();
for (int i = 0; i < m_items.Count; i++)
{
list.Add(m_items[i].CreateScenario(scenario));
}
SimpleMathSeries rval = new SimpleMathSeries(Name, list, m_operation);
rval.Name = this.Name;
rval.Appearance.LegendText = scenario.Name + " " + Name;
rval.ScenarioName = scenario.Name;
rval.SiteID = this.SiteID;
rval.TimeInterval = this.TimeInterval;
return(rval);
}
/// <summary>
/// gets daily dependents for this series (tablename)
/// </summary>
/// <param name="tableName"></param>
/// <param name="t"></param>
/// <returns></returns>
private SeriesList GetDailyDependents(string tableName)
{
SeriesList rval = new SeriesList();
TimeSeriesName tn = new TimeSeriesName(tableName);
var calcList = GetDependentCalculations(tableName, TimeInterval.Daily);
//if (calcList.Count > 0)
// Logger.WriteLine("Found " + calcList.Count + " daily calculations to update ref:"+tableName);
foreach (var item in calcList)
{
// prevent recursive?
rval.Add(item);
// check for daily that depends on daily.
var x = GetDailyDependents(item.Table.TableName);
rval.AddRange(x);
}
return(rval);
}
private static void PerformDailyComputations(SeriesList importSeries,
List <CalculationSeries> calculationQueue, SeriesList routingList)
{
// do Actual Computations now. (in proper order...)
TimeSeriesDependency td = new TimeSeriesDependency(calculationQueue);
TimeRange tr;
bool validRange = TryGetDailyTimeRange(importSeries, out tr, DateTime.Now);
if (!validRange)
{
Console.WriteLine(" time range indicates don't perform calculation.");
Console.WriteLine(" Current Time:" + DateTime.Now.ToString());
Console.WriteLine(" Default time range :" + tr.StartDate.ToString() + " " + tr.EndDate.ToString());
}
var sortedCalculations = td.Sort();
foreach (CalculationSeries cs in sortedCalculations)
{
Console.Write(">>> " + cs.Table.TableName + ": " + cs.Expression);
if (validRange)
{
cs.Calculate(tr.StartDate, tr.EndDate);
if (cs.Count > 0)
{
routingList.Add(cs);
if (cs.CountMissing() > 0)
{
Console.WriteLine(" Missing " + cs.CountMissing() + " records");
}
else
{
Console.WriteLine(" OK");
}
}
}
else
{
Console.WriteLine("Skipping because there is not a valid time range.");
}
}
}
public SimpleMathSeries(string name, SeriesList items, MathOperation[] op) : base()
{
this.Name = name;
m_items = items;
this.Appearance.LegendText = name;
this.SiteID = name;
m_operation = op;
if (items.Count == 0)
{
throw new ArgumentException("Must have at least 1 series to perform math");
}
if (items.Count != op.Length + 1)
{
throw new ArgumentException("Number of operators (+, -) must agree with number of series");
}
this.Units = items[0].Units;
this.TimeInterval = items[0].TimeInterval;
ValidateCalculation();
//= "SeriesList=";
var idList = new List <string>();
var opList = new List <string>();
for (int i = 0; i < items.Count; i++)
{
idList.Add(items[i].ID.ToString());
if (i < op.Length)
{
opList.Add(op[i].ToString());
}
}
ConnectionString = "SeriesList=" + String.Join(",", idList.ToArray())
+ ";OperatorList=" + String.Join(",", opList.ToArray())
+ ";LastUpdate="
+ DateTime.Now.ToString(DateTimeFormatInstantaneous);
Provider = "SimpleMathSeries";
}
