private void WriteListToFile(SeriesList list, string name, TimeInterval interval, string route) { var tmpFileName = FileUtility.GetTempFileName(".txt"); File.Delete(tmpFileName); Logger.WriteLine("writing " + list.Count + " series "); Logger.WriteLine("temp file:" + tmpFileName); if (interval == TimeInterval.Daily) { SeriesList exportList = new SeriesList(); foreach (var s in list) { exportList.Add(s); } HydrometDailySeries.WriteToArcImportFile(exportList, tmpFileName); } if (interval == TimeInterval.Irregular) { foreach (var s in list) { HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, Environment.UserName, tmpFileName, true); } } if (File.Exists(tmpFileName)) { Logger.WriteLine("Moving: " + tmpFileName); string fn = interval == TimeInterval.Daily ? "daily" : "instant"; var fileName = GetOutgoingFileName(fn, name, "all", route); Logger.WriteLine("To: " + fileName); File.Move(tmpFileName, fileName); } }
public void Add(SeriesList items) { foreach (Series s in items) { Add(s); } }
/// <summary> /// Imports list of series /// </summary> /// <param name="importSeries"></param> /// <param name="computeDependencies"></param> /// <param name="importTag"></param> /// <returns>list of series including dependent calculations</returns> private SeriesList Import(SeriesList importSeries, bool computeDependencies, string importTag) { Performance p = new Performance(); int calculationCount = 0; var routingList = new SeriesList(); foreach (var s in importSeries) { m_db.Quality.SetFlags(s); // to do, log/email flaged data m_db.ImportSeriesUsingTableName(s, m_saveOption); CheckForAlarms(s); routingList.Add(s); if (computeDependencies) { var z = ComputeDependenciesSameInterval(s); calculationCount += z.Count; foreach (var cs in z) { CheckForAlarms(cs); } routingList.AddRange(z); } } Export(importTag, routingList); Stats(importSeries.Count(), p.ElapsedSeconds, calculationCount, "instant"); return(routingList); }
private SeriesList ComputeDependenciesSameInterval(Series s) { int level = 0; SeriesList rval = new SeriesList(); var calcList = GetDependentsRecursive(s.Table.TableName, s.TimeInterval, ref level); if (calcList.Count > 0) { Logger.WriteLine("Found " + calcList.Count + " " + s.TimeInterval + " calculations to update "); } TimeSeriesDependency d = new TimeSeriesDependency(calcList); var sorted = d.Sort(); foreach (var item in sorted) { var cs = item as CalculationSeries; cs.Calculate(s.MinDateTime, s.MaxDateTime); if (cs.Count > 0) { rval.Add(cs); } } return(rval); }
/// <summary> /// Routes a list of Series as a group /// hydromet cbtt is copied from list[i].SiteName /// hydromet pcode is copied from list[i].Parameter /// </summary> /// <param name="list"></param> /// <param name="route"></param> /// <param name="name">identity used as part of filename </param> public static void RouteDaily(SeriesList list, string name, RouteOptions route = RouteOptions.Both) { if (list.Count == 0) { return; } string fileName = ""; if (route == RouteOptions.Both || route == RouteOptions.Outgoing) { fileName = GetOutgoingFileName("daily", name, "all"); Console.WriteLine("saving daily outgoing to:" + fileName); HydrometDailySeries.WriteToArcImportFile(list, fileName); } if (route == RouteOptions.Both || route == RouteOptions.Incoming) { foreach (var s in list) { fileName = GetIncommingFileName("daily", s.SiteID, s.Parameter); Console.WriteLine("saving daily incoming to:" + fileName); s.WriteCsv(fileName, true); } } }
private List <CalculationSeries> GetDailyDependentCalculations(SeriesList list) { Logger.WriteLine("GetDailyDependentCalculations(" + list.Count + " series )"); var rval = new List <CalculationSeries>(); foreach (var s in list) { int level = 0; if (NeedDailyCalc(s)) { // daily calcs that depend on instant var x = GetDependentsRecursive(s.Table.TableName, TimeInterval.Daily, ref level); if (x.Count > 0) { Logger.WriteLine("for " + s.Table.TableName + " found " + x.Count + " dependencies (before checking for duplicates)"); } foreach (var item in x) { // prevent duplicates if (!rval.Any(a => a.Table.TableName == item.Table.TableName)) { rval.Add(item); Logger.WriteLine(" --> " + item.Table.TableName); } } } } return(rval); }
/// <summary> /// Routes a list of Series as a group /// hydromet cbtt is copied from list[i].SiteName /// hydromet pcode is copied from list[i].Parameter /// </summary> /// <param name="list"></param> /// <param name="route"></param> /// <param name="name">identity used as part of filename </param> public static void RouteInstant(SeriesList list, string name, RouteOptions route = RouteOptions.Both) { if (route == RouteOptions.None) { return; } if (route == RouteOptions.Both || route == RouteOptions.Outgoing) { var tmpFileName = FileUtility.GetTempFileName(".txt"); File.Delete(tmpFileName); Console.WriteLine("temp file:" + tmpFileName); foreach (var s in list) { HydrometInstantSeries.WriteToHydrometFile(s, s.SiteID, s.Parameter, WindowsUtility.GetShortUserName(), tmpFileName, true); } Console.WriteLine("Moving: " + tmpFileName); var fileName = GetOutgoingFileName("instant", name, "all"); Console.WriteLine("To: " + fileName); File.Move(tmpFileName, fileName); } else { throw new NotImplementedException("incoming not supported."); } }
public Series Sum(DateTime t1, DateTime t2) { SeriesList sl = new SeriesList(); sl = this.Subset(t1, t2); return(sl.Sum()); }
/// <summary> /// Imports time series data, /// 1) set flags /// 2) active alarms (TO DO) /// 3) compute dependent data (same interval) /// 4) compute daily data when encountering midnight values /// </summary> /// <param name="importSeries"></param> /// <param name="computeDependencies"></param> /// <param name="computeDailyDependencies"></param> public void Import(SeriesList importSeries, bool computeDependencies = false, bool computeDailyDependencies = false, string importTag = "data") { var calculationQueue = new List <CalculationSeries>(); var routingList = new SeriesList(); foreach (var s in importSeries) { m_db.Quality.SetFlags(s); // to do, log/email flaged data ProcessAlarms(s); var folderNames = SetupDefaultFolders(s); m_db.ImportSeriesUsingTableName(s, folderNames, m_saveOption); routingList.Add(s); if (computeDependencies) { var z = ComputeDependenciesSameInterval(s); routingList.AddRange(z); } if (computeDailyDependencies && NeedDailyCalc(s)) { // daily calcs that depend on instant GetDailyDependentCalculations(s, calculationQueue); } } if (calculationQueue.Count > 0) { PerformDailyComputations(importSeries, calculationQueue, routingList); } RouteData(importTag, routingList); }
private SeriesList GetDailyCalculationsIfMidnight(Series s) { var calcList = new SeriesList(); // check for midnight values, and initiate daily calculations. if (s.TimeInterval == TimeInterval.Irregular) { for (int i = 0; i < s.Count; i++) { var pt = s[i]; if (pt.DateTime.IsMidnight()) { var x = GetDailyDependents(s.Table.TableName); foreach (var item in x) { if (!calcList.ContainsTableName(item)) { calcList.AddRange(x); } } } } } return(calcList); }
public SimpleMathSeries(TimeSeriesDatabase db, Reclamation.TimeSeries.TimeSeriesDatabaseDataSet.SeriesCatalogRow sr) : base(db, sr) { string[] idList = ConnectionStringUtility.GetToken(ConnectionString, "SeriesList", "").Split(','); string[] opList = ConnectionStringUtility.GetToken(ConnectionString, "OperatorList", "").Split(','); m_operation = new MathOperation[opList.Length]; m_items = new SeriesList(); for (int i = 0; i < idList.Length; i++) { int id = -1; if (!Int32.TryParse(idList[i], out id) || !db.SeriesExists(id)) { Logger.WriteLine("Cannot calculate series. Has the data this calculation needs been deleted?"); Name = "Error: source data missing " + Name; m_sourceDeleted = true; break; } else { Series s = db.GetSeries(id); m_items.Add(s); } } ScenarioName = m_items[0].ScenarioName; // TO DO assuming all scenarioNames are the same. for (int i = 0; i < opList.Length; i++) { m_operation[i] = MathOperationFromString(opList[i]); } }
/// <summary> /// Imports time series data, /// 1) set flags /// 2) active alarms (TO DO) /// 3) compute dependent data (same interval) /// 4) compute daily data when encountering midnight values /// </summary> /// <param name="importSeries"></param> /// <param name="computeDependencies"></param> /// <param name="computeDailyDependencies"></param> public void Import(SeriesList importSeries, bool computeDependencies = false, bool computeDailyDependencies = false, string importTag = "data") { var calculationQueue = new List <CalculationSeries>(); var routingList = new SeriesList(); foreach (var s in importSeries) { m_db.Quality.SetFlags(s); // to do, log/email flaged data m_db.Alarms.Check(s); // check for alarms; send email make phone calls var folderNames = new string[] {}; // TO DO //hydromet/cbtt default. m_db.ImportSeriesUsingTableName(s, folderNames, m_saveOption); routingList.Add(s); if (computeDependencies) { var z = ComputeDependenciesSameInterval(s); routingList.AddRange(z); } if (computeDailyDependencies && NeedDailyCalc(s)) { // daily calcs that depend on instant GetDailyDependentCalculations(s, calculationQueue); } } if (calculationQueue.Count > 0) { PerformDailyComputations(importSeries, calculationQueue, routingList); } RouteData(importTag, routingList); }
/// <summary> /// Export data to outgoing directory /// </summary> /// <param name="importTag"></param> /// <param name="routingList"></param> private void Export(string importTag, SeriesList routingList) { SeriesList instantRoute = new SeriesList(); SeriesList dailyRoute = new SeriesList(); // route data to other locations. foreach (var item in routingList) { TimeSeriesName tn = new TimeSeriesName(item.Table.TableName); item.Parameter = tn.pcode; item.SiteID = tn.siteid; if (item.TimeInterval == TimeInterval.Irregular) { instantRoute.Add(item); } if (item.TimeInterval == TimeInterval.Daily) { dailyRoute.Add(item); } } Logger.WriteLine("Exporting data"); TimeSeriesTransfer export = new TimeSeriesTransfer(m_db); export.Export(instantRoute, importTag, TimeInterval.Irregular); export.Export(dailyRoute, importTag, TimeInterval.Daily); }
/// <summary> /// Imports a single series (used for testing) /// </summary> /// <param name="s"></param> /// <param name="computeDependencies"></param> /// <param name="computeDailyEachMidnight"></param> internal void Import(Series s, bool computeDependencies = false, bool computeDailyEachMidnight = false) { var sl = new SeriesList(); sl.Add(s); Import(sl, computeDependencies, computeDailyEachMidnight, "test"); }
public SeriesList AggregateAndSubset(StatisticalMethods aggregateType, MonthDayRange monthDayRange, int beginningMonth) { SeriesList rval = new SeriesList(); foreach (Series s in this) { rval.Add(TimeSeries.Math.AggregateAndSubset(aggregateType, s, monthDayRange, beginningMonth)); } return(rval); }
public SeriesList Subset(MonthDayRange range) { SeriesList rval = new SeriesList(); foreach (Series s in this) { rval.Add(TimeSeries.Math.Subset(s, range)); } return(rval); }
public SeriesList Subset(int[] months) { SeriesList rval = new SeriesList(); foreach (Series s in this) { rval.Add(TimeSeries.Math.Subset(s, months)); } return(rval); }
List <int> m_columnToSeries; // index to series from each column in table /// <summary> /// Creates a DataTable composed from multiple time series /// </summary> public SeriesListDataTable(SeriesList list, TimeInterval interval) : base(interval.ToString()) { m_seriesList = list.FilteredList(interval); m_columnToSeries = new List <int>(); CreateMultiColumnSchema(); AddData(); AcceptChanges(); ColumnChanged += new DataColumnChangeEventHandler(m_table_ColumnChanged); SetupPrimaryKey(); }
/// <summary> /// Imports time series data, /// 1) set flags /// 2) active alarms /// 3) compute dependent data (same interval) /// 4) compute daily data when encountering midnight values /// </summary> /// <param name="importSeries"></param> /// <param name="computeDependencies"></param> /// <param name="computeDailyDependencies"></param> public void Import(SeriesList importSeries, bool computeDependencies = false, bool computeDailyDependencies = false, string importTag = "data") { var list = Import(importSeries, computeDependencies, importTag); if (computeDailyDependencies) { DailyCalculations(list, importTag); } }
public SeriesList SummaryHydrograph(int[] exceedanceLevels, DateTime beginningDate, bool max, bool min, bool avg, bool removeEmptySeries)//, bool removeFeb29) { SeriesList rval = new SeriesList(); foreach (Series s in this) { SeriesList list = Math.SummaryHydrograph(s, exceedanceLevels, beginningDate, max, min, avg, removeEmptySeries);//,removeFeb29); rval.AddRange(list); } return(rval); }
public static Series MLRInterpolationPisces(double fitTolerance, params Series[] s) { SeriesList sList = new SeriesList(); foreach (var item in s) { sList.Add(item); } var sOut = Reclamation.TimeSeries.Estimation.MultipleLinearRegression.MlrInterpolation(sList, new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, fitTolerance); return(sOut.EstimatedSeries); }
///// <summary> ///// Subtracts Series within a list from series in another list ///// </summary> ///// <param name="a"></param> ///// <param name="b"></param> ///// <returns></returns> //public static SeriesList operator -(SeriesList a, SeriesList b) //{ // if (a.Count != b.Count) // throw new ArgumentException("Error: number of series is not the same in the two lists"); // var rval = new SeriesList(); // for (int i = 0; i < a.Count; i++) // { // var sa = a[i]; // var sb = b[i]; // rval.Add(Math.Subtract(sa, sb)); // } // return rval; //} ///// <summary> ///// Adds Series within a list to series in another list ///// </summary> ///// <param name="a"></param> ///// <param name="b"></param> ///// <returns></returns> //public static SeriesList operator +(SeriesList a, SeriesList b) //{ // if (a.Count != b.Count) // throw new ArgumentException("Error: number of series is not the same in the two lists"); // var rval = new SeriesList(); // for (int i = 0; i < a.Count; i++) // { // var sa = a[i]; // var sb = b[i]; // rval.Add(Math.Add(sa, sb)); // } // return rval; //} /// <summary> /// Creates a new SeriesList Filtered by interval /// </summary> /// <param name="interval"></param> /// <returns></returns> public SeriesList FilteredList(TimeInterval interval) { var rval = new SeriesList(); foreach (var item in this) { if (item.TimeInterval == interval) { rval.Add(item); } } return(rval); }
public SeriesList Subset(DateTime d1, DateTime d2) { string datecol = this[0].Table.Columns[0].ColumnName; SeriesList rval = new SeriesList(); String sql = "[" + datecol + "] >= '" + d1.ToShortDateString() + "' AND [" + datecol + "] <= '" + d2.ToShortDateString() + "'"; foreach (Series s in this) { Series ss = s.Subset(sql); rval.Add(ss); } return(rval); }
/// <summary> /// Routes a list of Series as a group /// hydromet cbtt is copied from list[i].SiteName /// hydromet pcode is copied from list[i].Parameter /// </summary> /// <param name="list"></param> /// <param name="route"></param> /// <param name="name">identity used as part of filename </param> public void Export(SeriesList list, string name, TimeInterval interval) { if (list.Count == 0) { return; } var routes = RouteDestinations(list); foreach (string route in routes.Keys) { WriteListToFile(routes[route], name, interval, route); } }
private void DailyCalculations(SeriesList importSeries, string importTag) { Logger.WriteLine("DailyCalculations(" + importSeries.Count + " series, " + importTag + ")"); Performance p = new Performance(); var dailyCalculationQueue = GetDailyDependentCalculations(importSeries); var routingList = new SeriesList(); if (dailyCalculationQueue.Count > 0) { PerformDailyComputations(importSeries, dailyCalculationQueue, routingList); } Export(importTag, routingList); Stats(0, p.ElapsedSeconds, dailyCalculationQueue.Count, "daily"); }
private static void AddToRoute(Dictionary <string, SeriesList> rval, Series s, string key) { SeriesList sl = null; if (!rval.ContainsKey(key)) { sl = new SeriesList(); rval.Add(key, sl); } else { sl = rval[key]; } sl.Add(s); }
public override Series CreateScenario(TimeSeriesDatabaseDataSet.ScenarioRow scenario) { SeriesList list = new SeriesList(); for (int i = 0; i < m_items.Count; i++) { list.Add(m_items[i].CreateScenario(scenario)); } SimpleMathSeries rval = new SimpleMathSeries(Name, list, m_operation); rval.Name = this.Name; rval.Appearance.LegendText = scenario.Name + " " + Name; rval.ScenarioName = scenario.Name; rval.SiteID = this.SiteID; rval.TimeInterval = this.TimeInterval; return(rval); }
/// <summary> /// gets daily dependents for this series (tablename) /// </summary> /// <param name="tableName"></param> /// <param name="t"></param> /// <returns></returns> private SeriesList GetDailyDependents(string tableName) { SeriesList rval = new SeriesList(); TimeSeriesName tn = new TimeSeriesName(tableName); var calcList = GetDependentCalculations(tableName, TimeInterval.Daily); //if (calcList.Count > 0) // Logger.WriteLine("Found " + calcList.Count + " daily calculations to update ref:"+tableName); foreach (var item in calcList) { // prevent recursive? rval.Add(item); // check for daily that depends on daily. var x = GetDailyDependents(item.Table.TableName); rval.AddRange(x); } return(rval); }
private static void PerformDailyComputations(SeriesList importSeries, List <CalculationSeries> calculationQueue, SeriesList routingList) { // do Actual Computations now. (in proper order...) TimeSeriesDependency td = new TimeSeriesDependency(calculationQueue); TimeRange tr; bool validRange = TryGetDailyTimeRange(importSeries, out tr, DateTime.Now); if (!validRange) { Console.WriteLine(" time range indicates don't perform calculation."); Console.WriteLine(" Current Time:" + DateTime.Now.ToString()); Console.WriteLine(" Default time range :" + tr.StartDate.ToString() + " " + tr.EndDate.ToString()); } var sortedCalculations = td.Sort(); foreach (CalculationSeries cs in sortedCalculations) { Console.Write(">>> " + cs.Table.TableName + ": " + cs.Expression); if (validRange) { cs.Calculate(tr.StartDate, tr.EndDate); if (cs.Count > 0) { routingList.Add(cs); if (cs.CountMissing() > 0) { Console.WriteLine(" Missing " + cs.CountMissing() + " records"); } else { Console.WriteLine(" OK"); } } } else { Console.WriteLine("Skipping because there is not a valid time range."); } } }
public SimpleMathSeries(string name, SeriesList items, MathOperation[] op) : base() { this.Name = name; m_items = items; this.Appearance.LegendText = name; this.SiteID = name; m_operation = op; if (items.Count == 0) { throw new ArgumentException("Must have at least 1 series to perform math"); } if (items.Count != op.Length + 1) { throw new ArgumentException("Number of operators (+, -) must agree with number of series"); } this.Units = items[0].Units; this.TimeInterval = items[0].TimeInterval; ValidateCalculation(); //= "SeriesList="; var idList = new List <string>(); var opList = new List <string>(); for (int i = 0; i < items.Count; i++) { idList.Add(items[i].ID.ToString()); if (i < op.Length) { opList.Add(op[i].ToString()); } } ConnectionString = "SeriesList=" + String.Join(",", idList.ToArray()) + ";OperatorList=" + String.Join(",", opList.ToArray()) + ";LastUpdate=" + DateTime.Now.ToString(DateTimeFormatInstantaneous); Provider = "SimpleMathSeries"; }