internal void WriteVariableWarnings() { double mn; string expr; string fltr; string col; string yrcol = EnPIResources.yearColName; string blyr = ModelYear; foreach (DataColumn dc in SourceData.Columns) { if (IndependentVariables.Contains(dc.ColumnName)) { fltr = yrcol + "='" + blyr + "'"; col = "[" + dc.ColumnName.Replace("]", "\\]") + "]"; expr = "MIN(" + col + ")"; double min = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "MAX(" + col + ")"; double max = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "AVG(" + col + ")"; double mean = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "COUNT(" + col + ")"; double ct = double.Parse(SourceData.Compute(expr, fltr).ToString()); if (ct > 1) //stdev method fails with only one row { expr = "STDEV(" + col + ")"; // standard deviation computed using (n-1) method double stdev = double.Parse(SourceData.Compute(expr, fltr).ToString()); double low = Math.Min(mean - 3 * stdev, min); double high = Math.Max(mean + 3 * stdev, max); expr = "AVG(" + col + ")"; foreach (string yr in Years) { fltr = "[" + yrcol + "]='" + yr + "'"; mn = double.Parse(SourceData.Compute(expr, fltr).ToString()); if (mn < low || mn > high) { DataRow dr = VariableWarnings.NewRow(); dr[0] = dc.ColumnName; dr[1] = yr; VariableWarnings.Rows.Add(dr); } } } } } }
public int OutlierCount() { int ct = 0; //double mn; string expr; string fltr; string col; string yrcol = EnPIResources.yearColName; string blyr = ModelYear; foreach (DataColumn dc in SourceData.Columns) { if (IndependentVariables.Contains(dc.ColumnName)) { fltr = yrcol + "='" + blyr + "'"; col = "[" + dc.ColumnName.Replace("]", "\\]") + "]"; expr = "COUNT(" + col + ")"; double basect = double.Parse(SourceData.Compute(expr, fltr).ToString()); if (basect > 1) //can't compute stdev with one point { expr = "MIN(" + col + ")"; double min = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "MAX(" + col + ")"; double max = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "AVG(" + col + ")"; double mean = double.Parse(SourceData.Compute(expr, fltr).ToString()); expr = "STDEV(" + col + ")"; // standard deviation computed using (n-1) method double stdev = double.Parse(SourceData.Compute(expr, fltr).ToString()); double low = Math.Min(mean - 3 * stdev, min); double high = Math.Max(mean + 3 * stdev, max); expr = col + " < " + low.ToString() + " OR " + col + " > " + high.ToString(); ct += SourceData.Select(expr).Count(); } } } return(ct); }
internal void ExcludeBlanks() { bool modelnulls = false; bool othernulls = false; string fltr; string col; string yrcol = EnPIResources.yearColName; string blyr = ModelYear; foreach (DataColumn dc in SourceData.Columns) { if (IndependentVariables.Contains(dc.ColumnName) || EnergySourceVariables.Contains(dc.ColumnName)) { col = "[" + dc.ColumnName.Replace("]", "\\]") + "]"; // remove rows with missing values from model year fltr = yrcol + "='" + blyr + "' and " + col + " is null"; foreach (DataRow dr in SourceData.Select(fltr)) { SourceData.Rows.Remove(dr); modelnulls = true; } // remove rows with missing values from other years fltr = yrcol + "<>'" + blyr + "' and " + col + " is null"; foreach (DataRow dr in SourceData.Select(fltr)) { SourceData.Rows.Remove(dr); othernulls = true; } } if (modelnulls) { DataRow vr = VariableWarnings.NewRow(); vr[2] = "Rows with blank values were excluded from the model"; VariableWarnings.Rows.Add(vr); } if (othernulls) { DataRow vr = VariableWarnings.NewRow(); vr[2] = "Rows with blank values were excluded from the results"; VariableWarnings.Rows.Add(vr); } } }
public void Init()//bool fromRegression { if (ModelYear != null) { ExcludeBlanks(); } // set model data string yrcol = EnPIResources.yearColName; string fltr = yrcol + "='" + ModelYear + "'"; ModelData = SourceData.Copy(); if (ModelYear != null) //replace all data with just the data for the model year { ModelData.Clear(); foreach (DataRow dr in SourceData.Select(fltr)) { ModelData.ImportRow(dr); } if (ModelData.Rows.Count < Utilities.Constants.MODEL_MIN_DATAPOINTS) { DataRow vr = VariableWarnings.NewRow(); vr[2] = "Selected model year contains less than " + Utilities.Constants.MODEL_MIN_DATAPOINTS.ToString() + " data points"; VariableWarnings.Rows.Add(vr); } } // set knownXs and knownYs knownXs = ModelData.Copy(); knownYs = ModelData.Copy(); foreach (DataColumn dc in ModelData.Columns) { if (!EnergySourceVariables.Contains(dc.ColumnName)) { knownYs.Columns.Remove(dc.ColumnName); } if (!IndependentVariables.Contains(dc.ColumnName)) { knownXs.Columns.Remove(dc.ColumnName); } } if (EnergySourceVariables.Contains(EnPIResources.unadjustedTotalColName)) { knownYs = DataHelper.AddSumColumn(knownYs); } if (EnergySourceVariables != null) { // this will create the collection of energy sources and the list of IVs foreach (string col in EnergySourceVariables) { EnergySource aSource = new EnergySource(col); aSource.knownXs = knownXs; try { double dcol = Convert.ToDouble(col); string col1 = dcol.ToString("#,###0"); aSource.Ys = Ys(col1); } catch { aSource.Ys = Ys(col); } aSource.Combinations = AllCombinations(); aSource.AddModels(); EnergySources.Add(aSource); } } WriteVariableWarnings(); }