/// <summary>
/// Run tests
/// </summary>
/// <param name="accept">If true, the stats from this run will be written to file as the accepted stats.</param>
/// <param name="GUIrun">If true, do not raise an exception on test failure.</param>
public void Test(bool accept = false, bool GUIrun = false)
{
PredictedObserved PO = Parent as PredictedObserved;
if (PO == null)
return;
DataStore DS = PO.Parent as DataStore;
MathUtilities.RegrStats[] stats;
List<string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available
DataTable POtable = DS.GetData("*", PO.Name);
List<string> columnNames;
string sigIdent = "X";
if (POtable == null)
{
object sim = PO.Parent;
while (sim as Simulations == null)
sim = ((Model)sim).Parent;
throw new ApsimXException(this, "Could not find PO table in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>") + ". Has the simulation been run?");
}
columnNames = POtable.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names
columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs
for (int i = 0; i < columnNames.Count; i++)
columnNames[i] = columnNames[i].Replace("Observed.", "");
columnNames.Sort(); //ensure column names are always in the same order
stats = new MathUtilities.RegrStats[columnNames.Count];
List<double> x = new List<double>();
List<double> y = new List<double>();
string xstr, ystr;
double xres;
double yres;
for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair
{
x.Clear();
y.Clear();
foreach (DataRow row in POtable.Rows)
{
xstr = row["Observed." + columnNames[c]].ToString();
ystr = row["Predicted." + columnNames[c]].ToString();
if (Double.TryParse(xstr, out xres) && Double.TryParse(ystr, out yres))
{
x.Add(xres);
y.Add(yres);
}
}
if (x.Count == 0 || y.Count == 0)
continue;
stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x);
}
//remove any null stats which can occur from non-numeric columns such as dates
List<MathUtilities.RegrStats> list = new List<MathUtilities.RegrStats>(stats);
list.RemoveAll(l => l == null);
stats = list.ToArray();
//remove entries from column names
for (int i = columnNames.Count() - 1; i >= 0; i--)
{
bool found = false;
for (int j = 0; j < stats.Count(); j++)
{
if (columnNames[i] == stats[j].Name)
{
found = true;
break;
}
}
if (!found)
columnNames.RemoveAt(i);
}
//turn stats array into a DataTable
//first, check if there is already an AcceptedStats array, create if not.
//If the names don't match, then use current stats as user has dragged
//an already existing Test to a new node.
if (AcceptedStats == null || POName != PO.Name)
{
POName = PO.Name;
AcceptedStats = stats;
AcceptedStatsName = StringUtilities.Build(statNames, " ");
}
//then make sure the names and order of the accepted stats are the same as the new ones.
if (StringUtilities.Build(statNames, " ") != AcceptedStatsName)
throw new ApsimXException(this, "Names, number or order of accepted stats do not match class MathUtilities.RegrStats. The class has probably changed.");
Table = new DataTable("StatTests");
Table.Columns.Add("Name", typeof(string));
Table.Columns.Add("Variable", typeof(string));
Table.Columns.Add("Test", typeof(string));
Table.Columns.Add("Accepted", typeof(double));
Table.Columns.Add("Current", typeof(double));
Table.Columns.Add("Difference", typeof(double));
Table.Columns.Add("Fail?", typeof(string));
double accepted;
double current;
DataTable AcceptedTable = Table.Copy();
DataTable CurrentTable = Table.Copy();
//accepted table
for (int i = 0; i < AcceptedStats.Count(); i++)
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
accepted = Convert.ToDouble(AcceptedStats[i].GetType().GetField(statNames[j]).GetValue(AcceptedStats[i]));
AcceptedTable.Rows.Add(PO.Name,
AcceptedStats[i].Name,
statNames[j],
accepted,
null,
null,
null);
}
//current table
Table = AcceptedTable.Copy();
int rowIndex = 0;
for (int i = 0; i < stats.Count(); i++)
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
current = Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]));
CurrentTable.Rows.Add(PO.Name,
stats[i].Name,
statNames[j],
null,
current,
null,
null);
Table.Rows[rowIndex]["Current"] = current;
rowIndex++;
}
//Merge overwrites rows, so add the correct data back in
foreach(DataRow row in Table.Rows)
{
DataRow[] rowAccepted = AcceptedTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
DataRow[] rowCurrent = CurrentTable.Select ("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
if (rowAccepted.Count() == 0)
row["Accepted"] = DBNull.Value;
else
row["Accepted"] = rowAccepted[0]["Accepted"];
if (rowCurrent.Count() == 0)
row["Current"] = DBNull.Value;
else
row["Current"] = rowCurrent[0]["Current"];
if (row["Accepted"] != DBNull.Value && row["Current"] != DBNull.Value)
{
row["Difference"] = Convert.ToDouble(row["Current"]) - Convert.ToDouble(row["Accepted"]);
row["Fail?"] = Math.Abs(Convert.ToDouble(row["Difference"])) > Math.Abs(Convert.ToDouble(row["Accepted"])) * 0.01 ? sigIdent : " ";
}
else
{
row["Difference"] = DBNull.Value;
row["Fail?"] = sigIdent;
}
}
//Tables could be large so free the memory.
AcceptedTable = null;
CurrentTable = null;
if (accept)
AcceptedStats = stats;
else
{
foreach (DataRow row in Table.Rows)
if (row["Fail?"].ToString().Equals(sigIdent))
{
if (!GUIrun)
{
object sim = PO.Parent;
while (sim as Simulations == null)
sim = ((Model)sim).Parent;
throw new ApsimXException(this, "Significant differences found during regression testing of " + PO.Name + " in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>"));
}
}
}
}
public static DataTable CalculateStatsOnPredictedObservedValues(DataTable POtable)
{
DataTable currentTable = CreateTestsStatsTable();
try
{
Utilities.WriteToLogFile(" 1/4. CalculateStatsOnPredictedObservedValues: Start processing");
MathUtilities.RegrStats[] stats;
List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available
var columnNames = (from row in POtable.AsEnumerable() select row.Field <string>("ValueName")).Distinct().ToList();
//columnNames.Sort(); //ensure column names are always in the same order
stats = new MathUtilities.RegrStats[columnNames.Count];
List <double> x = new List <double>();
List <double> y = new List <double>();
string valueName = string.Empty, xstr, ystr;
int c = 0;
string holdValueName = POtable.Rows[0]["ValueName"].ToString();
//loop through our current POtable and collate the necessary data to calculate stats for current PredictedObservedValues
foreach (DataRow row in POtable.Rows) //on each P/O column pair
{
valueName = row["ValueName"].ToString();
if (valueName != holdValueName)
{
if (x.Count != 0 || y.Count != 0)
{
stats[c] = MathUtilities.CalcRegressionStats(holdValueName, y, x);
c += 1;
}
holdValueName = valueName;
x.Clear();
y.Clear();
}
Double xres, yres;
xstr = row["ObservedValue"].ToString();
ystr = row["PredictedValue"].ToString();
if ((Double.TryParse(xstr, out xres)) && (Double.TryParse(ystr, out yres)))
{
x.Add(xres);
y.Add(yres);
}
}
Utilities.WriteToLogFile(" 2/4. CalculateStatsOnPredictedObservedValues: CalcRegressionStats");
if (x.Count != 0 || y.Count != 0)
{
stats[c] = MathUtilities.CalcRegressionStats(holdValueName, y, x);
}
//remove any null stats which can occur from non-numeric columns such as dates
List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats);
list.RemoveAll(l => l == null);
stats = list.ToArray();
//remove entries from column names
for (int i = columnNames.Count() - 1; i >= 0; i--)
{
bool found = false;
for (int j = 0; j < stats.Count(); j++)
{
if (columnNames[i].ToString() == stats[j].Name)
{
found = true;
break;
}
}
if (!found)
{
columnNames.RemoveAt(i);
}
}
double current;
DataRow tRow;
bool hasValue;
string variable, test, statValue, helperStr;
Utilities.WriteToLogFile(" 3/4. CalculateStatsOnPredictedObservedValues: Loop through stats and put them into a datatable");
//Loop through stats and put them into a datatable
int rowIndex = 0;
for (int i = 0; i < stats.Count(); i++)
{
//for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
for (int j = 0; j < statNames.Count; j++) //need to ensure wthat we dont do 'Name'
{
//current = Math.Round(Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i])), 6);
//CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null);
test = statNames[j];
if (test != "Name")
{
variable = stats[i].Name;
statValue = stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]).ToString();
helperStr = "Variable: " + variable + ", Test: " + test + ", Value: " + statValue;
//CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null);
tRow = currentTable.NewRow();
tRow["Variable"] = variable;
tRow["Test"] = test;
hasValue = true;
try
{
current = Math.Round(Convert.ToDouble(statValue), 6);
if (double.IsNaN(current) == true)
{
hasValue = false;
}
if (double.IsInfinity(current) == true)
{
hasValue = false;
}
if (hasValue == true)
{
tRow["Current"] = current;
//currentTable.Rows[rowIndex]["Current"] = current;
}
}
catch (Exception)
{
Utilities.WriteToLogFile(" ERROR in DoValidationTest: Unable to convert:" + helperStr);
}
currentTable.Rows.Add(tRow);
rowIndex++;
}
}
}
Utilities.WriteToLogFile(" 4/4. CalculateStatsOnPredictedObservedValues: completed");
}
catch (Exception ex)
{
//throw new Exception("ERROR in CalculateStatsOnPredictedObservedValues: Unable to process Test Data: " + ex.Message.ToString());
Utilities.WriteToLogFile(" ERROR in CalculateStatsOnPredictedObservedValues: Unable to process Test Data: " + ex.Message.ToString());
}
return(currentTable);
}
/// <summary>
/// Run tests
/// </summary>
/// <param name="accept">If true, the stats from this run will be written to file as the accepted stats.</param>
/// <param name="GUIrun">If true, do not raise an exception on test failure.</param>
public void Test(bool accept = false, bool GUIrun = false)
{
PredictedObserved PO = Parent as PredictedObserved;
if (PO == null)
{
return;
}
DataStore DS = PO.Parent as DataStore;
MathUtilities.RegrStats[] stats;
List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available
DataTable POtable = DS.GetData("*", PO.Name);
List <string> columnNames;
string sigIdent = "X";
if (POtable == null)
{
object sim = PO.Parent;
while (sim as Simulations == null)
{
sim = ((Model)sim).Parent;
}
throw new ApsimXException(this, "Could not find PO table in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>") + ". Has the simulation been run?");
}
columnNames = POtable.Columns.Cast <DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names
columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs
for (int i = 0; i < columnNames.Count; i++)
{
columnNames[i] = columnNames[i].Replace("Observed.", "");
}
columnNames.Sort(); //ensure column names are always in the same order
stats = new MathUtilities.RegrStats[columnNames.Count];
List <double> x = new List <double>();
List <double> y = new List <double>();
string xstr, ystr;
double xres;
double yres;
for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair
{
x.Clear();
y.Clear();
foreach (DataRow row in POtable.Rows)
{
xstr = row["Observed." + columnNames[c]].ToString();
ystr = row["Predicted." + columnNames[c]].ToString();
if (Double.TryParse(xstr, out xres) && Double.TryParse(ystr, out yres))
{
x.Add(xres);
y.Add(yres);
}
}
if (x.Count == 0 || y.Count == 0)
{
continue;
}
stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x);
}
//remove any null stats which can occur from non-numeric columns such as dates
List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats);
list.RemoveAll(l => l == null);
stats = list.ToArray();
//remove entries from column names
for (int i = columnNames.Count() - 1; i >= 0; i--)
{
bool found = false;
for (int j = 0; j < stats.Count(); j++)
{
if (columnNames[i] == stats[j].Name)
{
found = true;
break;
}
}
if (!found)
{
columnNames.RemoveAt(i);
}
}
//turn stats array into a DataTable
//first, check if there is already an AcceptedStats array, create if not.
//If the names don't match, then use current stats as user has dragged
//an already existing Test to a new node.
if (AcceptedStats == null || POName != PO.Name)
{
POName = PO.Name;
AcceptedStats = stats;
AcceptedStatsName = StringUtilities.Build(statNames, " ");
}
//then make sure the names and order of the accepted stats are the same as the new ones.
if (StringUtilities.Build(statNames, " ") != AcceptedStatsName)
{
throw new ApsimXException(this, "Names, number or order of accepted stats do not match class MathUtilities.RegrStats. The class has probably changed.");
}
Table = new DataTable("StatTests");
Table.Columns.Add("Name", typeof(string));
Table.Columns.Add("Variable", typeof(string));
Table.Columns.Add("Test", typeof(string));
Table.Columns.Add("Accepted", typeof(double));
Table.Columns.Add("Current", typeof(double));
Table.Columns.Add("Difference", typeof(double));
Table.Columns.Add("Fail?", typeof(string));
double accepted;
double current;
DataTable AcceptedTable = Table.Copy();
DataTable CurrentTable = Table.Copy();
//accepted table
for (int i = 0; i < AcceptedStats.Count(); i++)
{
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
accepted = Convert.ToDouble(AcceptedStats[i].GetType().GetField(statNames[j]).GetValue(AcceptedStats[i]));
AcceptedTable.Rows.Add(PO.Name,
AcceptedStats[i].Name,
statNames[j],
accepted,
null,
null,
null);
}
}
//current table
Table = AcceptedTable.Copy();
int rowIndex = 0;
for (int i = 0; i < stats.Count(); i++)
{
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
current = Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]));
CurrentTable.Rows.Add(PO.Name,
stats[i].Name,
statNames[j],
null,
current,
null,
null);
Table.Rows[rowIndex]["Current"] = current;
rowIndex++;
}
}
//Merge overwrites rows, so add the correct data back in
foreach (DataRow row in Table.Rows)
{
DataRow[] rowAccepted = AcceptedTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
DataRow[] rowCurrent = CurrentTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
if (rowAccepted.Count() == 0)
{
row["Accepted"] = DBNull.Value;
}
else
{
row["Accepted"] = rowAccepted[0]["Accepted"];
}
if (rowCurrent.Count() == 0)
{
row["Current"] = DBNull.Value;
}
else
{
row["Current"] = rowCurrent[0]["Current"];
}
if (row["Accepted"] != DBNull.Value && row["Current"] != DBNull.Value)
{
row["Difference"] = Convert.ToDouble(row["Current"]) - Convert.ToDouble(row["Accepted"]);
row["Fail?"] = Math.Abs(Convert.ToDouble(row["Difference"])) > Math.Abs(Convert.ToDouble(row["Accepted"])) * 0.01 ? sigIdent : " ";
}
else
{
row["Difference"] = DBNull.Value;
row["Fail?"] = sigIdent;
}
}
//Tables could be large so free the memory.
AcceptedTable = null;
CurrentTable = null;
if (accept)
{
AcceptedStats = stats;
}
else
{
foreach (DataRow row in Table.Rows)
{
if (row["Fail?"].ToString().Equals(sigIdent))
{
if (!GUIrun)
{
object sim = PO.Parent;
while (sim as Simulations == null)
{
sim = ((Model)sim).Parent;
}
throw new ApsimXException(this, "Significant differences found during regression testing of " + PO.Name + " in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>"));
}
}
}
}
}
public const double Threshold = 0.01; // 1%
/// <summary>
/// Run tests
/// </summary>
public static DataTable DoValidationTest(string PO_Name, DataTable POtable, DataTable acceptedStats)
{
DataTable currentTable = CreateTestsStatsTable();
string helperStr = string.Empty;
try
{
MathUtilities.RegrStats[] stats;
List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available
List <string> columnNames;
Utilities.WriteToLogFile(" 1/8. DoValidationTest: get the column names");
columnNames = POtable.Columns.Cast <DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names
columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs
for (int i = 0; i < columnNames.Count; i++)
{
columnNames[i] = columnNames[i].Replace("Observed.", "");
}
columnNames.Sort(); //ensure column names are always in the same order
stats = new MathUtilities.RegrStats[columnNames.Count];
List <double> x = new List <double>();
List <double> y = new List <double>();
string xstr, ystr;
Double xres, yres;
Utilities.WriteToLogFile(" 2/8. DoValidationTest: get the predicted observed values and calc regression stats");
for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair
{
x.Clear();
y.Clear();
foreach (DataRow row in POtable.Rows)
{
xstr = row["Observed." + columnNames[c]].ToString();
ystr = row["Predicted." + columnNames[c]].ToString();
if ((Double.TryParse(xstr, out xres)) && (Double.TryParse(ystr, out yres)))
{
x.Add(xres);
y.Add(yres);
}
}
if (x.Count == 0 || y.Count == 0)
{
continue;
}
stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x);
}
//remove any null stats which can occur from non-numeric columns such as dates
Utilities.WriteToLogFile(" 3/8. DoValidationTest: remove any null stats which can occur from non-numeric columns such as dates");
List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats);
list.RemoveAll(l => l == null);
stats = list.ToArray();
//remove entries from column names
Utilities.WriteToLogFile(" 4/8. DoValidationTest: remove entries from column names");
for (int i = columnNames.Count() - 1; i >= 0; i--)
{
bool found = false;
for (int j = 0; j < stats.Count(); j++)
{
if (columnNames[i] == stats[j].Name)
{
found = true;
break;
}
}
if (!found)
{
columnNames.RemoveAt(i);
}
}
double current;
DataRow tRow;
bool hasValue;
string variable, test, statValue;
//Loop through stats and put them into a datatable
Utilities.WriteToLogFile(" 5/8. DoValidationTest: Loop through stats and put them into a datatable ");
int rowIndex = 0;
for (int i = 0; i < stats.Count(); i++)
{
//for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
for (int j = 0; j < statNames.Count; j++) //need to ensure we don't do 'Name'
{
test = statNames[j];
if (test != "Name")
{
variable = stats[i].Name;
statValue = stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]).ToString();
helperStr = "Variable: " + variable + ", Test: " + test + ", Value: " + statValue;
//CurrentTable.Rows.Add(PO_Name, stats[i].Name, statNames[j], null, current, null, null);
tRow = currentTable.NewRow();
tRow["Variable"] = variable;
tRow["Test"] = test;
hasValue = true;
try
{
current = Math.Round(Convert.ToDouble(statValue), 6);
if (double.IsNaN(current) == true)
{
hasValue = false;
}
if (double.IsInfinity(current) == true)
{
hasValue = false;
}
if (hasValue == true)
{
tRow["Current"] = current;
//currentTable.Rows[rowIndex]["Current"] = current;
}
}
catch (Exception)
{
Utilities.WriteToLogFile(" ERROR in DoValidationTest: Unable to convert:" + helperStr);
}
currentTable.Rows.Add(tRow);
rowIndex++;
}
}
}
helperStr = string.Empty;
Utilities.WriteToLogFile(" 6/8. DoValidationTest: Loop through stats and put them into a datatable - Completed");
//Now merge this with out Accepted Table
//Now add the comparison columns and determine values
Utilities.WriteToLogFile(" 7/8. DoValidationTest: MergeAndCompareAcceptedAgainstCurrent ");
MergeAndCompareAcceptedAgainstCurrent(ref currentTable, acceptedStats);
//Need to ensure that the order of the columns in the Datatable matches our table type
Utilities.WriteToLogFile(" 8/8. DoValidationTest: OrderCurrentTableforTableType ");
OrderCurrentTableforTableType(ref currentTable);
Utilities.WriteToLogFile(" DoValidationTest: complete");
}
catch (Exception ex)
{
//throw new Exception("ERROR in DoValidationTest:: " + ex.Message.ToString());
Utilities.WriteToLogFile(" ERROR in DoValidationTest: " + helperStr + " - " + ex.Message.ToString());
}
return(currentTable);
}