/// <summary> /// Run tests /// </summary> /// <param name="accept">If true, the stats from this run will be written to file as the accepted stats.</param> /// <param name="GUIrun">If true, do not raise an exception on test failure.</param> public void Test(bool accept = false, bool GUIrun = false) { PredictedObserved PO = Parent as PredictedObserved; if (PO == null) return; DataStore DS = PO.Parent as DataStore; MathUtilities.RegrStats[] stats; List<string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available DataTable POtable = DS.GetData("*", PO.Name); List<string> columnNames; string sigIdent = "X"; if (POtable == null) { object sim = PO.Parent; while (sim as Simulations == null) sim = ((Model)sim).Parent; throw new ApsimXException(this, "Could not find PO table in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>") + ". Has the simulation been run?"); } columnNames = POtable.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs for (int i = 0; i < columnNames.Count; i++) columnNames[i] = columnNames[i].Replace("Observed.", ""); columnNames.Sort(); //ensure column names are always in the same order stats = new MathUtilities.RegrStats[columnNames.Count]; List<double> x = new List<double>(); List<double> y = new List<double>(); string xstr, ystr; double xres; double yres; for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair { x.Clear(); y.Clear(); foreach (DataRow row in POtable.Rows) { xstr = row["Observed." + columnNames[c]].ToString(); ystr = row["Predicted." + columnNames[c]].ToString(); if (Double.TryParse(xstr, out xres) && Double.TryParse(ystr, out yres)) { x.Add(xres); y.Add(yres); } } if (x.Count == 0 || y.Count == 0) continue; stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x); } //remove any null stats which can occur from non-numeric columns such as dates List<MathUtilities.RegrStats> list = new List<MathUtilities.RegrStats>(stats); list.RemoveAll(l => l == null); stats = list.ToArray(); //remove entries from column names for (int i = columnNames.Count() - 1; i >= 0; i--) { bool found = false; for (int j = 0; j < stats.Count(); j++) { if (columnNames[i] == stats[j].Name) { found = true; break; } } if (!found) columnNames.RemoveAt(i); } //turn stats array into a DataTable //first, check if there is already an AcceptedStats array, create if not. //If the names don't match, then use current stats as user has dragged //an already existing Test to a new node. if (AcceptedStats == null || POName != PO.Name) { POName = PO.Name; AcceptedStats = stats; AcceptedStatsName = StringUtilities.Build(statNames, " "); } //then make sure the names and order of the accepted stats are the same as the new ones. if (StringUtilities.Build(statNames, " ") != AcceptedStatsName) throw new ApsimXException(this, "Names, number or order of accepted stats do not match class MathUtilities.RegrStats. The class has probably changed."); Table = new DataTable("StatTests"); Table.Columns.Add("Name", typeof(string)); Table.Columns.Add("Variable", typeof(string)); Table.Columns.Add("Test", typeof(string)); Table.Columns.Add("Accepted", typeof(double)); Table.Columns.Add("Current", typeof(double)); Table.Columns.Add("Difference", typeof(double)); Table.Columns.Add("Fail?", typeof(string)); double accepted; double current; DataTable AcceptedTable = Table.Copy(); DataTable CurrentTable = Table.Copy(); //accepted table for (int i = 0; i < AcceptedStats.Count(); i++) for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. { accepted = Convert.ToDouble(AcceptedStats[i].GetType().GetField(statNames[j]).GetValue(AcceptedStats[i])); AcceptedTable.Rows.Add(PO.Name, AcceptedStats[i].Name, statNames[j], accepted, null, null, null); } //current table Table = AcceptedTable.Copy(); int rowIndex = 0; for (int i = 0; i < stats.Count(); i++) for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. { current = Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i])); CurrentTable.Rows.Add(PO.Name, stats[i].Name, statNames[j], null, current, null, null); Table.Rows[rowIndex]["Current"] = current; rowIndex++; } //Merge overwrites rows, so add the correct data back in foreach(DataRow row in Table.Rows) { DataRow[] rowAccepted = AcceptedTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'"); DataRow[] rowCurrent = CurrentTable.Select ("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'"); if (rowAccepted.Count() == 0) row["Accepted"] = DBNull.Value; else row["Accepted"] = rowAccepted[0]["Accepted"]; if (rowCurrent.Count() == 0) row["Current"] = DBNull.Value; else row["Current"] = rowCurrent[0]["Current"]; if (row["Accepted"] != DBNull.Value && row["Current"] != DBNull.Value) { row["Difference"] = Convert.ToDouble(row["Current"]) - Convert.ToDouble(row["Accepted"]); row["Fail?"] = Math.Abs(Convert.ToDouble(row["Difference"])) > Math.Abs(Convert.ToDouble(row["Accepted"])) * 0.01 ? sigIdent : " "; } else { row["Difference"] = DBNull.Value; row["Fail?"] = sigIdent; } } //Tables could be large so free the memory. AcceptedTable = null; CurrentTable = null; if (accept) AcceptedStats = stats; else { foreach (DataRow row in Table.Rows) if (row["Fail?"].ToString().Equals(sigIdent)) { if (!GUIrun) { object sim = PO.Parent; while (sim as Simulations == null) sim = ((Model)sim).Parent; throw new ApsimXException(this, "Significant differences found during regression testing of " + PO.Name + " in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>")); } } } }
public static DataTable CalculateStatsOnPredictedObservedValues(DataTable POtable) { DataTable currentTable = CreateTestsStatsTable(); try { Utilities.WriteToLogFile(" 1/4. CalculateStatsOnPredictedObservedValues: Start processing"); MathUtilities.RegrStats[] stats; List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available var columnNames = (from row in POtable.AsEnumerable() select row.Field <string>("ValueName")).Distinct().ToList(); //columnNames.Sort(); //ensure column names are always in the same order stats = new MathUtilities.RegrStats[columnNames.Count]; List <double> x = new List <double>(); List <double> y = new List <double>(); string valueName = string.Empty, xstr, ystr; int c = 0; string holdValueName = POtable.Rows[0]["ValueName"].ToString(); //loop through our current POtable and collate the necessary data to calculate stats for current PredictedObservedValues foreach (DataRow row in POtable.Rows) //on each P/O column pair { valueName = row["ValueName"].ToString(); if (valueName != holdValueName) { if (x.Count != 0 || y.Count != 0) { stats[c] = MathUtilities.CalcRegressionStats(holdValueName, y, x); c += 1; } holdValueName = valueName; x.Clear(); y.Clear(); } Double xres, yres; xstr = row["ObservedValue"].ToString(); ystr = row["PredictedValue"].ToString(); if ((Double.TryParse(xstr, out xres)) && (Double.TryParse(ystr, out yres))) { x.Add(xres); y.Add(yres); } } Utilities.WriteToLogFile(" 2/4. CalculateStatsOnPredictedObservedValues: CalcRegressionStats"); if (x.Count != 0 || y.Count != 0) { stats[c] = MathUtilities.CalcRegressionStats(holdValueName, y, x); } //remove any null stats which can occur from non-numeric columns such as dates List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats); list.RemoveAll(l => l == null); stats = list.ToArray(); //remove entries from column names for (int i = columnNames.Count() - 1; i >= 0; i--) { bool found = false; for (int j = 0; j < stats.Count(); j++) { if (columnNames[i].ToString() == stats[j].Name) { found = true; break; } } if (!found) { columnNames.RemoveAt(i); } } double current; DataRow tRow; bool hasValue; string variable, test, statValue, helperStr; Utilities.WriteToLogFile(" 3/4. CalculateStatsOnPredictedObservedValues: Loop through stats and put them into a datatable"); //Loop through stats and put them into a datatable int rowIndex = 0; for (int i = 0; i < stats.Count(); i++) { //for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. for (int j = 0; j < statNames.Count; j++) //need to ensure wthat we dont do 'Name' { //current = Math.Round(Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i])), 6); //CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null); test = statNames[j]; if (test != "Name") { variable = stats[i].Name; statValue = stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]).ToString(); helperStr = "Variable: " + variable + ", Test: " + test + ", Value: " + statValue; //CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null); tRow = currentTable.NewRow(); tRow["Variable"] = variable; tRow["Test"] = test; hasValue = true; try { current = Math.Round(Convert.ToDouble(statValue), 6); if (double.IsNaN(current) == true) { hasValue = false; } if (double.IsInfinity(current) == true) { hasValue = false; } if (hasValue == true) { tRow["Current"] = current; //currentTable.Rows[rowIndex]["Current"] = current; } } catch (Exception) { Utilities.WriteToLogFile(" ERROR in DoValidationTest: Unable to convert:" + helperStr); } currentTable.Rows.Add(tRow); rowIndex++; } } } Utilities.WriteToLogFile(" 4/4. CalculateStatsOnPredictedObservedValues: completed"); } catch (Exception ex) { //throw new Exception("ERROR in CalculateStatsOnPredictedObservedValues: Unable to process Test Data: " + ex.Message.ToString()); Utilities.WriteToLogFile(" ERROR in CalculateStatsOnPredictedObservedValues: Unable to process Test Data: " + ex.Message.ToString()); } return(currentTable); }
/// <summary> /// Run tests /// </summary> /// <param name="accept">If true, the stats from this run will be written to file as the accepted stats.</param> /// <param name="GUIrun">If true, do not raise an exception on test failure.</param> public void Test(bool accept = false, bool GUIrun = false) { PredictedObserved PO = Parent as PredictedObserved; if (PO == null) { return; } DataStore DS = PO.Parent as DataStore; MathUtilities.RegrStats[] stats; List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available DataTable POtable = DS.GetData("*", PO.Name); List <string> columnNames; string sigIdent = "X"; if (POtable == null) { object sim = PO.Parent; while (sim as Simulations == null) { sim = ((Model)sim).Parent; } throw new ApsimXException(this, "Could not find PO table in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>") + ". Has the simulation been run?"); } columnNames = POtable.Columns.Cast <DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs for (int i = 0; i < columnNames.Count; i++) { columnNames[i] = columnNames[i].Replace("Observed.", ""); } columnNames.Sort(); //ensure column names are always in the same order stats = new MathUtilities.RegrStats[columnNames.Count]; List <double> x = new List <double>(); List <double> y = new List <double>(); string xstr, ystr; double xres; double yres; for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair { x.Clear(); y.Clear(); foreach (DataRow row in POtable.Rows) { xstr = row["Observed." + columnNames[c]].ToString(); ystr = row["Predicted." + columnNames[c]].ToString(); if (Double.TryParse(xstr, out xres) && Double.TryParse(ystr, out yres)) { x.Add(xres); y.Add(yres); } } if (x.Count == 0 || y.Count == 0) { continue; } stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x); } //remove any null stats which can occur from non-numeric columns such as dates List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats); list.RemoveAll(l => l == null); stats = list.ToArray(); //remove entries from column names for (int i = columnNames.Count() - 1; i >= 0; i--) { bool found = false; for (int j = 0; j < stats.Count(); j++) { if (columnNames[i] == stats[j].Name) { found = true; break; } } if (!found) { columnNames.RemoveAt(i); } } //turn stats array into a DataTable //first, check if there is already an AcceptedStats array, create if not. //If the names don't match, then use current stats as user has dragged //an already existing Test to a new node. if (AcceptedStats == null || POName != PO.Name) { POName = PO.Name; AcceptedStats = stats; AcceptedStatsName = StringUtilities.Build(statNames, " "); } //then make sure the names and order of the accepted stats are the same as the new ones. if (StringUtilities.Build(statNames, " ") != AcceptedStatsName) { throw new ApsimXException(this, "Names, number or order of accepted stats do not match class MathUtilities.RegrStats. The class has probably changed."); } Table = new DataTable("StatTests"); Table.Columns.Add("Name", typeof(string)); Table.Columns.Add("Variable", typeof(string)); Table.Columns.Add("Test", typeof(string)); Table.Columns.Add("Accepted", typeof(double)); Table.Columns.Add("Current", typeof(double)); Table.Columns.Add("Difference", typeof(double)); Table.Columns.Add("Fail?", typeof(string)); double accepted; double current; DataTable AcceptedTable = Table.Copy(); DataTable CurrentTable = Table.Copy(); //accepted table for (int i = 0; i < AcceptedStats.Count(); i++) { for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. { accepted = Convert.ToDouble(AcceptedStats[i].GetType().GetField(statNames[j]).GetValue(AcceptedStats[i])); AcceptedTable.Rows.Add(PO.Name, AcceptedStats[i].Name, statNames[j], accepted, null, null, null); } } //current table Table = AcceptedTable.Copy(); int rowIndex = 0; for (int i = 0; i < stats.Count(); i++) { for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. { current = Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i])); CurrentTable.Rows.Add(PO.Name, stats[i].Name, statNames[j], null, current, null, null); Table.Rows[rowIndex]["Current"] = current; rowIndex++; } } //Merge overwrites rows, so add the correct data back in foreach (DataRow row in Table.Rows) { DataRow[] rowAccepted = AcceptedTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'"); DataRow[] rowCurrent = CurrentTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'"); if (rowAccepted.Count() == 0) { row["Accepted"] = DBNull.Value; } else { row["Accepted"] = rowAccepted[0]["Accepted"]; } if (rowCurrent.Count() == 0) { row["Current"] = DBNull.Value; } else { row["Current"] = rowCurrent[0]["Current"]; } if (row["Accepted"] != DBNull.Value && row["Current"] != DBNull.Value) { row["Difference"] = Convert.ToDouble(row["Current"]) - Convert.ToDouble(row["Accepted"]); row["Fail?"] = Math.Abs(Convert.ToDouble(row["Difference"])) > Math.Abs(Convert.ToDouble(row["Accepted"])) * 0.01 ? sigIdent : " "; } else { row["Difference"] = DBNull.Value; row["Fail?"] = sigIdent; } } //Tables could be large so free the memory. AcceptedTable = null; CurrentTable = null; if (accept) { AcceptedStats = stats; } else { foreach (DataRow row in Table.Rows) { if (row["Fail?"].ToString().Equals(sigIdent)) { if (!GUIrun) { object sim = PO.Parent; while (sim as Simulations == null) { sim = ((Model)sim).Parent; } throw new ApsimXException(this, "Significant differences found during regression testing of " + PO.Name + " in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>")); } } } } }
public const double Threshold = 0.01; // 1% /// <summary> /// Run tests /// </summary> public static DataTable DoValidationTest(string PO_Name, DataTable POtable, DataTable acceptedStats) { DataTable currentTable = CreateTestsStatsTable(); string helperStr = string.Empty; try { MathUtilities.RegrStats[] stats; List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available List <string> columnNames; Utilities.WriteToLogFile(" 1/8. DoValidationTest: get the column names"); columnNames = POtable.Columns.Cast <DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs for (int i = 0; i < columnNames.Count; i++) { columnNames[i] = columnNames[i].Replace("Observed.", ""); } columnNames.Sort(); //ensure column names are always in the same order stats = new MathUtilities.RegrStats[columnNames.Count]; List <double> x = new List <double>(); List <double> y = new List <double>(); string xstr, ystr; Double xres, yres; Utilities.WriteToLogFile(" 2/8. DoValidationTest: get the predicted observed values and calc regression stats"); for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair { x.Clear(); y.Clear(); foreach (DataRow row in POtable.Rows) { xstr = row["Observed." + columnNames[c]].ToString(); ystr = row["Predicted." + columnNames[c]].ToString(); if ((Double.TryParse(xstr, out xres)) && (Double.TryParse(ystr, out yres))) { x.Add(xres); y.Add(yres); } } if (x.Count == 0 || y.Count == 0) { continue; } stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x); } //remove any null stats which can occur from non-numeric columns such as dates Utilities.WriteToLogFile(" 3/8. DoValidationTest: remove any null stats which can occur from non-numeric columns such as dates"); List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats); list.RemoveAll(l => l == null); stats = list.ToArray(); //remove entries from column names Utilities.WriteToLogFile(" 4/8. DoValidationTest: remove entries from column names"); for (int i = columnNames.Count() - 1; i >= 0; i--) { bool found = false; for (int j = 0; j < stats.Count(); j++) { if (columnNames[i] == stats[j].Name) { found = true; break; } } if (!found) { columnNames.RemoveAt(i); } } double current; DataRow tRow; bool hasValue; string variable, test, statValue; //Loop through stats and put them into a datatable Utilities.WriteToLogFile(" 5/8. DoValidationTest: Loop through stats and put them into a datatable "); int rowIndex = 0; for (int i = 0; i < stats.Count(); i++) { //for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field. for (int j = 0; j < statNames.Count; j++) //need to ensure we don't do 'Name' { test = statNames[j]; if (test != "Name") { variable = stats[i].Name; statValue = stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]).ToString(); helperStr = "Variable: " + variable + ", Test: " + test + ", Value: " + statValue; //CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null); tRow = currentTable.NewRow(); tRow["Variable"] = variable; tRow["Test"] = test; hasValue = true; try { current = Math.Round(Convert.ToDouble(statValue), 6); if (double.IsNaN(current) == true) { hasValue = false; } if (double.IsInfinity(current) == true) { hasValue = false; } if (hasValue == true) { tRow["Current"] = current; //currentTable.Rows[rowIndex]["Current"] = current; } } catch (Exception) { Utilities.WriteToLogFile(" ERROR in DoValidationTest: Unable to convert:" + helperStr); } currentTable.Rows.Add(tRow); rowIndex++; } } } helperStr = string.Empty; Utilities.WriteToLogFile(" 6/8. DoValidationTest: Loop through stats and put them into a datatable - Completed"); //Now merge this with out Accepted Table //Now add the comparison columns and determine values Utilities.WriteToLogFile(" 7/8. DoValidationTest: MergeAndCompareAcceptedAgainstCurrent "); MergeAndCompareAcceptedAgainstCurrent(ref currentTable, acceptedStats); //Need to ensure that the order of the columns in the Datatable matches our table type Utilities.WriteToLogFile(" 8/8. DoValidationTest: OrderCurrentTableforTableType "); OrderCurrentTableforTableType(ref currentTable); Utilities.WriteToLogFile(" DoValidationTest: complete"); } catch (Exception ex) { //throw new Exception("ERROR in DoValidationTest:: " + ex.Message.ToString()); Utilities.WriteToLogFile(" ERROR in DoValidationTest: " + helperStr + " - " + ex.Message.ToString()); } return(currentTable); }