public static void TestCommentsInEventNames()
{
Simulations file = Utilities.GetRunnableSim();
Report report = Apsim.Find(file, typeof(Report)) as Report;
report.Name = "Report"; // Just to make sure
report.VariableNames = new string[] { "[Clock].Today.DayOfYear as doy" };
report.EventNames = new string[]
{
"[Clock].StartOfWeek // works normally",
"// Should be ignored",
"//[Clock].EndOfWeek // entire line should be ignored"
};
Clock clock = Apsim.Find(file, typeof(Clock)) as Clock;
clock.StartDate = new DateTime(2017, 1, 1);
clock.EndDate = new DateTime(2017, 3, 1);
Runner runner = new Runner(file);
List <Exception> errors = runner.Run();
if (errors != null && errors.Count > 0)
{
throw errors[0];
}
List <string> fieldNames = new List <string>()
{
"doy"
};
IDataStore storage = Apsim.Find(file, typeof(IDataStore)) as IDataStore;
DataTable data = storage.Reader.GetData("Report", fieldNames: fieldNames);
double[] actual = DataTableUtilities.GetColumnAsDoubles(data, "doy");
double[] expected = new double[] { 1, 8, 15, 22, 29, 36, 43, 50, 57 };
Assert.AreEqual(expected, actual);
}
/// <summary>This refreshes data being displayed on the graphs, based on the value of the startYear and showYear values </summary>
/// <param name="table">The data set</param>
/// <param name="startDate">The start date</param>
/// <param name="endDate">The end date</param>
/// <param name="updateYears">Update the years</param>
private void DisplayGraphRadiation(DataTable table, DateTime startDate, DateTime endDate, bool updateYears)
{
try
{
if (table != null && table.Rows.Count > 0)
{
DateTime[] dailyDates = DataTableUtilities.GetColumnAsDates(table, "Date", startDate, endDate);
double[] dailyRain = DataTableUtilities.GetColumnAsDoubles(table, "rain", startDate, endDate);
double[] dailyRadn = DataTableUtilities.GetColumnAsDoubles(table, "radn", startDate, endDate);
double[] dailyMaxRadn = DataTableUtilities.GetColumnAsDoubles(table, "Qmax", startDate, endDate);
if (dailyRadn.Length != 0)
{
this.PopulateRadiationGraph("Radiation", dailyDates, dailyRain, dailyRadn, dailyMaxRadn);
}
}
}
catch (Exception e)
{
throw new Exception("Unable to display Detailed Graphs due to insufficient data: " + e.Message.ToString());
}
}
public void TestArrayRangeWithStartAndEndSpecification()
{
var mod = new MockModel()
{
Z = new double[] { 1, 2, 3 }
};
simulation.Children.Add(mod);
simulation.Children.Remove(storage);
var datastore = new DataStore();
simulation.Children.Add(datastore);
Utilities.InitialiseModel(simulation);
report.VariableNames = new string[] { "[MockModel].Z[2:3]" };
List <Exception> errors = runner.Run();
Assert.NotNull(errors);
Assert.AreEqual(0, errors.Count);
datastore.Writer.Stop();
datastore.Reader.Refresh();
var data = datastore.Reader.GetData("Report");
var columnNames = DataTableUtilities.GetColumnNames(data);
Assert.IsFalse(columnNames.Contains("MockModel.Z(0)"));
Assert.IsFalse(columnNames.Contains("MockModel.Z(1)"));
Assert.IsTrue(columnNames.Contains("MockModel.Z(2)"));
Assert.IsTrue(columnNames.Contains("MockModel.Z(3)"));
Assert.IsFalse(columnNames.Contains("MockModel.Z(4)"));
Assert.AreEqual(DataTableUtilities.GetColumnAsDoubles(data, "MockModel.Z(2)"),
new double[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 });
Assert.AreEqual(DataTableUtilities.GetColumnAsDoubles(data, "MockModel.Z(3)"),
new double[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 });
}
/// <summary>Gets a column of data from a view.</summary>
/// <param name="data">The table</param>
/// <param name="fieldName">Name of the field.</param>
/// <returns>The column of data.</returns>
private IEnumerable GetDataFromView(DataView data, string fieldName)
{
if (fieldName != null && data != null && data.Table.Columns.Contains(fieldName))
{
if (data.Table.Columns[fieldName].DataType == typeof(DateTime))
{
return(DataTableUtilities.GetColumnAsDates(data, fieldName));
}
else if (data.Table.Columns[fieldName].DataType == typeof(string))
{
return(DataTableUtilities.GetColumnAsStrings(data, fieldName));
}
else if (data.Table.Columns[fieldName].DataType == typeof(int))
{
return(DataTableUtilities.GetColumnAsIntegers(data, fieldName));
}
else
{
return(DataTableUtilities.GetColumnAsDoubles(data, fieldName));
}
}
return(null);
}
public static void TestReportingOnModelEvents()
{
string json = ReflectionUtilities.GetResourceAsString("UnitTests.Report.ReportOnEvents.apsimx");
Simulations file = FileFormat.ReadFromString <Simulations>(json, e => throw e, false);
// This simulation needs a weather node, but using a legit
// met component will just slow down the test.
IModel sim = file.FindInScope <Simulation>();
Model weather = new MockWeather();
sim.Children.Add(weather);
weather.Parent = sim;
// Run the file.
var Runner = new Runner(file);
Runner.Run();
// Check that the report reported on the correct dates.
var storage = file.FindInScope <IDataStore>();
List <string> fieldNames = new List <string>()
{
"doy"
};
DataTable data = storage.Reader.GetData("ReportOnFertilisation", fieldNames: fieldNames);
double[] values = DataTableUtilities.GetColumnAsDoubles(data, "doy");
double[] expected = new double[] { 1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 364 };
Assert.AreEqual(expected, values);
data = storage.Reader.GetData("ReportOnIrrigation", fieldNames: fieldNames);
values = DataTableUtilities.GetColumnAsDoubles(data, "doy");
// There is one less irrigation event, as the manager script doesn't irrigate.
expected = new double[] { 1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335 };
Assert.AreEqual(expected, values);
}
/// <summary>
/// Using the SimulationName column in the specified 'table', add a
/// SimulationID column.
/// </summary>
/// <param name="table">The table.</param>
/// <exception cref="Models.Core.ApsimXException">Cannot find Simulations table</exception>
private void AddSimulationIDColumnToTable(DataTable table)
{
DataTable idTable = Connection.ExecuteQuery("SELECT * FROM Simulations");
if (idTable == null)
{
throw new ApsimXException(this, "Cannot find Simulations table");
}
List <double> ids = new List <double>();
ids.AddRange(DataTableUtilities.GetColumnAsDoubles(idTable, "ID"));
List <string> simulationNames = new List <string>();
simulationNames.AddRange(DataTableUtilities.GetColumnAsStrings(idTable, "Name"));
table.Columns.Add("SimulationID", typeof(int)).SetOrdinal(0);
foreach (DataRow row in table.Rows)
{
string simulationName = row["SimulationName"].ToString();
if (simulationName != null)
{
int index = StringUtilities.IndexOfCaseInsensitive(simulationNames, simulationName);
if (index != -1)
{
row["SimulationID"] = ids[index];
}
else
{
int id = GetSimulationID(simulationName);
ids.Add(id);
simulationNames.Add(simulationName);
row["SimulationID"] = id;
}
}
}
}
public void RunAPSIMGetOutputs()
{
APSIMSpecification simulation = GetDefaultSimulationSpec();
List <APSIMSpecification> simulations = new List <APSIMSpecification>();
simulations.Add(simulation);
RuntimeEnvironment environment = new RuntimeEnvironment
{
APSIMRevision = "Apsim7.8-R4013"
};
RunYPJob job = new RunYPJob(simulations, environment);
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
// Make sure we don't have an error.
Assert.AreEqual(job.Errors.Count, 0);
// Make sure we have a daily output table.
Assert.AreEqual(job.Outputs.Tables.Count, 3);
Assert.AreEqual(job.Outputs.Tables[0].TableName, "Summary");
Assert.AreEqual(job.Outputs.Tables[1].TableName, "YieldProphetDaily");
Assert.AreEqual(job.Outputs.Tables[1].Rows.Count, 92);
double[] biomass = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[1], "biomass");
Assert.IsTrue(MathUtilities.Max(biomass) > 20.0); // make sure something is growing.
// Make sure we have a depth table.
Assert.AreEqual(job.Outputs.Tables[2].TableName, "YieldProphetDepth");
Assert.AreEqual(job.Outputs.Tables[2].Rows.Count, 8);
double[] sw = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[2], "sw");
Assert.IsTrue(MathUtilities.Max(sw) > 0.0); // make sure there are sw values
}
/// <summary>Main run method for performing our post simulation calculations</summary>
public void Run()
{
// If the predicted table has not been modified, don't do anything.
// This can happen if other simulations were run but the Morris model was not.
if (dataStore?.Writer != null && !dataStore.Writer.TablesModified.Contains(TableName))
{
return;
}
DataTable predictedData = dataStore.Reader.GetData(TableName);
if (predictedData != null)
{
// Determine how many aggregation values we have per simulation
DataView view = new DataView(predictedData);
view.RowFilter = "SimulationName='" + Name + "Simulation1'";
AggregationValues = DataTableUtilities.GetColumnAsStrings(view, AggregationVariableName);
// Create a table of all predicted values
DataTable predictedValues = new DataTable();
List <string> descriptiveColumnNames = new List <string>();
List <string> variableNames = new List <string>();
foreach (string aggregationValue in AggregationValues)
{
string value = aggregationValue;
if (DateTime.TryParse(value, out DateTime date))
{
value = date.ToString("yyyy-MM-dd");
}
view.RowFilter = $"{AggregationVariableName}='{value}'";
foreach (DataColumn predictedColumn in view.Table.Columns)
{
if (predictedColumn.DataType == typeof(double))
{
double[] values = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
if (values.Distinct().Count() == 1)
{
if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
{
descriptiveColumnNames.Add(predictedColumn.ColumnName);
}
}
else
{
DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + "_" + value, values);
if (!variableNames.Contains(predictedColumn.ColumnName))
{
variableNames.Add(predictedColumn.ColumnName);
}
}
}
}
}
// Run R
DataTable eeDataRaw;
DataTable statsDataRaw;
RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);
// Get ee data from R and store in ee table.
// EE data from R looks like:
// "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
// - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
// - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
// - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
// - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
DataView eeView = new DataView(eeDataRaw);
IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", AggregationVariableName });
// Create a path variable.
var pathValues = Enumerable.Range(1, NumPaths).ToArray();
foreach (var parameter in Parameters)
{
foreach (DataColumn column in predictedValues.Columns)
{
eeView.RowFilter = "variable = '" + column.ColumnName + "'";
if (eeView.Count != NumPaths)
{
throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
}
string aggregationValue = StringUtilities.GetAfter(column.ColumnName, "_");
string variableName = StringUtilities.RemoveAfter(column.ColumnName, '_');
eeTableKey.SetIndex(new object[] { parameter.Name, aggregationValue });
List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
for (int i = 0; i < values.Count; i++)
{
values[i] = Math.Abs(values[i]);
}
var runningMean = MathUtilities.RunningAverage(values);
eeTableKey.SetValues("Path", pathValues);
eeTableKey.SetValues(variableName + ".MuStar", runningMean);
}
}
DataTable eeTable = eeTableKey.ToTable();
eeTable.TableName = Name + "PathAnalysis";
// Get stats data from R and store in MuStar table.
// Stats data coming back from R looks like:
// "mu", "mustar", "sigma", "param","variable"
// -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
// -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
// -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
// 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
// -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
// 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
// 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
// -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
"Parameter", AggregationVariableName
});
foreach (DataRow row in statsDataRaw.Rows)
{
string variable = row["variable"].ToString();
string aggregationValue = StringUtilities.GetAfter(variable, "_");
variable = StringUtilities.RemoveAfter(variable, '_');
tableKey.SetIndex(new object[] { row["param"], aggregationValue });
tableKey.Set(variable + ".Mu", row["mu"]);
tableKey.Set(variable + ".MuStar", row["mustar"]);
tableKey.Set(variable + ".Sigma", row["sigma"]);
// Need to bring in the descriptive values.
view.RowFilter = $"{AggregationVariableName}='{aggregationValue}'";
foreach (var descriptiveColumnName in descriptiveColumnNames)
{
var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
if (values.Distinct().Count() == 1)
{
tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
}
}
}
DataTable muStarTable = tableKey.ToTable();
muStarTable.TableName = Name + "Statistics";
dataStore.Writer.WriteTable(eeTable);
dataStore.Writer.WriteTable(muStarTable);
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
if (PredictedTableName == null || ObservedTableName == null)
{
return;
}
// If neither the predicted nor obseved tables have been modified during
// the most recent simulations run, don't do anything.
if (dataStore?.Writer != null &&
!(dataStore.Writer.TablesModified.Contains(PredictedTableName) || dataStore.Writer.TablesModified.Contains(ObservedTableName)))
{
return;
}
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
List <string> commonCols = predictedDataNames.Intersect(observedDataNames).ToList();
// This should be all columns which exist in one table but not both.
IEnumerable <string> uncommonCols = predictedDataNames.Except(observedDataNames).Union(observedDataNames.Except(predictedDataNames));
IStorageReader reader = dataStore.Reader;
string match1ObsShort = reader.BriefColumnName(ObservedTableName, FieldNameUsedForMatch);
string match2ObsShort = reader.BriefColumnName(ObservedTableName, FieldName2UsedForMatch);
string match3ObsShort = reader.BriefColumnName(ObservedTableName, FieldName3UsedForMatch);
string match1PredShort = reader.BriefColumnName(PredictedTableName, FieldNameUsedForMatch);
string match2PredShort = reader.BriefColumnName(PredictedTableName, FieldName2UsedForMatch);
string match3PredShort = reader.BriefColumnName(PredictedTableName, FieldName3UsedForMatch);
StringBuilder query = new StringBuilder("SELECT ");
for (int i = 0; i < commonCols.Count; i++)
{
string s = commonCols[i];
string obsColShort = reader.BriefColumnName(ObservedTableName, s);
string predColShort = reader.BriefColumnName(PredictedTableName, s);
if (i != 0)
{
query.Append(", ");
}
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append($"O.\"{obsColShort}\"");
}
else
{
query.Append($"O.\"{obsColShort}\" AS \"Observed.{obsColShort}\", P.\"{predColShort}\" AS \"Predicted.{predColShort}\"");
}
}
// Add columns which exist in one table but not both.
foreach (string uncommonCol in uncommonCols)
{
// Basically this hack is here to allow error data to be added to the p/o graphs.
// This is kind of terrible, but I really don't want to duplicate every
// column from both predicted and observed tables if we don't have to.
// This does raise the question of whether we should be creating a "PredictedObserved"
// table at all, since we're actually duplicating data in the DB by doing so.
if (AllColumns || uncommonCol.EndsWith("Error"))
{
if (predictedDataNames.Contains(uncommonCol))
{
query.Append($", P.\"{uncommonCol}\" as \"Predicted.{uncommonCol}\"");
}
else if (observedDataNames.Contains(uncommonCol))
{
query.Append($", O.\"{uncommonCol}\" as \"Observed.{uncommonCol}\"");
}
}
}
query.AppendLine();
query.AppendLine($"FROM [{ObservedTableName}] O");
query.AppendLine($"INNER JOIN [{PredictedTableName}] P");
query.Append($"USING ([SimulationID], [CheckpointID], [{FieldNameUsedForMatch}]");
if (!string.IsNullOrEmpty(FieldName2UsedForMatch))
{
query.Append($", \"{FieldName2UsedForMatch}\"");
}
if (!string.IsNullOrEmpty(FieldName3UsedForMatch))
{
query.Append($", \"{FieldName3UsedForMatch}\"");
}
query.AppendLine(")");
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.AppendLine("WHERE [CheckpointID] = " + checkpointID);
query.Replace("O.\"SimulationID\" AS \"Observed.SimulationID\", P.\"SimulationID\" AS \"Predicted.SimulationID\"", "O.\"SimulationID\" AS \"SimulationID\"");
query.Replace("O.\"CheckpointID\" AS \"Observed.CheckpointID\", P.\"CheckpointID\" AS \"Predicted.CheckpointID\"", "O.\"CheckpointID\" AS \"CheckpointID\"");
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in this.FindAllInScope <Experiment>())
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in this.FindAllInScope <Simulation>())
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.[SimulationID] in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName, null));
}
query.Append(")");
}
DataTable predictedObservedData = reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
foreach (DataColumn column in predictedObservedData.Columns)
{
if (column.ColumnName.StartsWith("Predicted."))
{
string shortName = column.ColumnName.Substring("Predicted.".Length);
column.ColumnName = "Predicted." + reader.FullColumnName(PredictedTableName, shortName);
}
else if (column.ColumnName.StartsWith("Observed."))
{
string shortName = column.ColumnName.Substring("Observed.".Length);
column.ColumnName = "Observed." + reader.FullColumnName(ObservedTableName, shortName);
}
else if (column.ColumnName.Equals(match1ObsShort) || column.ColumnName.Equals(match2ObsShort) || column.ColumnName.Equals(match3ObsShort))
{
column.ColumnName = reader.FullColumnName(ObservedTableName, column.ColumnName);
}
}
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
reader.Refresh();
foreach (string fieldName in commonCols)
{
string units = reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
// The Writer replaces tables, rather than appends to them,
// so we actually need to re-write the existing units table values
// Is there a better way to do this?
DataView allUnits = new DataView(reader.GetData("_Units"));
allUnits.Sort = "TableName";
DataTable tableNames = allUnits.ToTable(true, "TableName");
foreach (DataRow row in tableNames.Rows)
{
string tableName = row["TableName"] as string;
List <string> colNames = new List <string>();
List <string> unitz = new List <string>();
foreach (DataRowView rowView in allUnits.FindRows(tableName))
{
colNames.Add(rowView["ColumnHeading"].ToString());
unitz.Add(rowView["Units"].ToString());
}
dataStore.Writer.AddUnits(tableName, colNames, unitz);
}
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = reader.GetDataUsingSql("SELECT * FROM [" + PredictedTableName + "]");
DataTable observedData = reader.GetDataUsingSql("SELECT * FROM [" + ObservedTableName + "]");
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
if (PredictedTableName != null && ObservedTableName != null)
{
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
IEnumerable <string> commonCols = predictedDataNames.Intersect(observedDataNames);
IStorageReader reader = dataStore.Reader;
string match1ObsShort = reader.BriefColumnName(ObservedTableName, FieldNameUsedForMatch);
string match2ObsShort = reader.BriefColumnName(ObservedTableName, FieldName2UsedForMatch);
string match3ObsShort = reader.BriefColumnName(ObservedTableName, FieldName3UsedForMatch);
string match1PredShort = reader.BriefColumnName(PredictedTableName, FieldNameUsedForMatch);
string match2PredShort = reader.BriefColumnName(PredictedTableName, FieldName2UsedForMatch);
string match3PredShort = reader.BriefColumnName(PredictedTableName, FieldName3UsedForMatch);
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
string obsColShort = reader.BriefColumnName(ObservedTableName, s);
string predColShort = reader.BriefColumnName(PredictedTableName, s);
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("O.[" + obsColShort + "], ");
}
else
{
query.Append("O.[" + obsColShort + "] AS [Observed." + obsColShort + "], P.[" + predColShort + "] AS [Predicted." + predColShort + "], ");
}
}
query.Append("FROM [" + ObservedTableName + "] O INNER JOIN [" + PredictedTableName + "] P USING ([SimulationID]) WHERE O.[" + match1ObsShort + "] = P.[" + match1PredShort + "]");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match2ObsShort + "] = P.[" + match2PredShort + "]");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match3ObsShort + "] = P.[" + match3PredShort + "]");
}
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.Append(" AND P.[CheckpointID] = " + checkpointID);
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("O.[SimulationID] AS [Observed.SimulationID], P.[SimulationID] AS [Predicted.SimulationID]", "O.[SimulationID] AS [SimulationID]");
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.[SimulationID] in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName, null));
}
query.Append(")");
}
DataTable predictedObservedData = reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
foreach (DataColumn column in predictedObservedData.Columns)
{
if (column.ColumnName.StartsWith("Predicted."))
{
string shortName = column.ColumnName.Substring("Predicted.".Length);
column.ColumnName = "Predicted." + reader.FullColumnName(PredictedTableName, shortName);
}
else if (column.ColumnName.StartsWith("Observed."))
{
string shortName = column.ColumnName.Substring("Observed.".Length);
column.ColumnName = "Observed." + reader.FullColumnName(ObservedTableName, shortName);
}
else if (column.ColumnName.Equals(match1ObsShort) || column.ColumnName.Equals(match2ObsShort) || column.ColumnName.Equals(match3ObsShort))
{
column.ColumnName = reader.FullColumnName(ObservedTableName, column.ColumnName);
}
}
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
reader.Refresh();
foreach (string fieldName in commonCols)
{
string units = reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
// The Writer replaces tables, rather than appends to them,
// so we actually need to re-write the existing units table values
// Is there a better way to do this?
DataView allUnits = new DataView(reader.GetData("_Units"));
allUnits.Sort = "TableName";
DataTable tableNames = allUnits.ToTable(true, "TableName");
foreach (DataRow row in tableNames.Rows)
{
string tableName = row["TableName"] as string;
List <string> colNames = new List <string>();
List <string> unitz = new List <string>();
foreach (DataRowView rowView in allUnits.FindRows(tableName))
{
colNames.Add(rowView["ColumnHeading"].ToString());
unitz.Add(rowView["Units"].ToString());
}
dataStore.Writer.AddUnits(tableName, colNames, unitz);
}
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = reader.GetDataUsingSql("SELECT * FROM [" + PredictedTableName + "]");
DataTable observedData = reader.GetDataUsingSql("SELECT * FROM [" + ObservedTableName + "]");
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
if (dataStore?.Writer != null && !dataStore.Writer.TablesModified.Contains("Report"))
{
return;
}
DataTable predictedData = dataStore.Reader.GetData("Report", filter: "SimulationName LIKE '" + Name + "%'", orderBy: "SimulationID");
if (predictedData != null)
{
IndexedDataTable variableValues = new IndexedDataTable(null);
// Determine how many years we have per simulation
DataView view = new DataView(predictedData);
view.RowFilter = "SimulationName='" + Name + "Simulation1'";
var Years = DataTableUtilities.GetColumnAsIntegers(view, "Clock.Today.Year");
// Create a results table.
IndexedDataTable results;
if (Years.Count() > 1)
{
results = new IndexedDataTable(new string[] { "Year" });
}
else
{
results = new IndexedDataTable(null);
}
// Loop through all years and perform analysis on each.
List <string> errorsFromR = new List <string>();
foreach (double year in Years)
{
view.RowFilter = "Clock.Today.Year=" + year;
foreach (DataColumn predictedColumn in predictedData.Columns)
{
if (predictedColumn.DataType == typeof(double))
{
var values = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
if (values.Distinct().Count() > 1)
{
variableValues.SetValues(predictedColumn.ColumnName, values);
}
}
}
string paramNames = StringUtilities.Build(Parameters.Select(p => p.Name), ",", "\"", "\"");
string sobolx1FileName = GetTempFileName("sobolx1", ".csv");
string sobolx2FileName = GetTempFileName("sobolx2", ".csv");
string sobolVariableValuesFileName = GetTempFileName("sobolvariableValues", ".csv");
// Write variables file
using (var writer = new StreamWriter(sobolVariableValuesFileName))
DataTableUtilities.DataTableToText(variableValues.ToTable(), 0, ",", true, writer, excelFriendly: false, decimalFormatString: "F6");
// Write X1
using (var writer = new StreamWriter(sobolx1FileName))
DataTableUtilities.DataTableToText(X1, 0, ",", true, writer, excelFriendly: false, decimalFormatString: "F6");
// Write X2
using (var writer = new StreamWriter(sobolx2FileName))
DataTableUtilities.DataTableToText(X2, 0, ",", true, writer, excelFriendly: false, decimalFormatString: "F6");
string script = string.Format(
$".libPaths(c('{R.PackagesDirectory}', .libPaths()))" + Environment.NewLine +
$"library('boot', lib.loc = '{R.PackagesDirectory}')" + Environment.NewLine +
$"library('sensitivity', lib.loc = '{R.PackagesDirectory}')" + Environment.NewLine +
"params <- c({0})" + Environment.NewLine +
"n <- {1}" + Environment.NewLine +
"nparams <- {2}" + Environment.NewLine +
"X1 <- read.csv(\"{3}\")" + Environment.NewLine +
"X2 <- read.csv(\"{4}\")" + Environment.NewLine +
"sa <- sobolSalt(model = NULL, X1, X2, scheme=\"A\", nboot = 100)" + Environment.NewLine +
"variableValues = read.csv(\"{5}\")" + Environment.NewLine +
"for (columnName in colnames(variableValues))" + Environment.NewLine +
"{{" + Environment.NewLine +
" sa$y <- variableValues[[columnName]]" + Environment.NewLine +
" tell(sa)" + Environment.NewLine +
" sa$S$Parameter <- params" + Environment.NewLine +
" sa$T$Parameter <- params" + Environment.NewLine +
" sa$S$ColumnName <- columnName" + Environment.NewLine +
" sa$T$ColumnName <- columnName" + Environment.NewLine +
" sa$S$Indices <- \"FirstOrder\"" + Environment.NewLine +
" sa$T$Indices <- \"Total\"" + Environment.NewLine +
" if (!exists(\"allData\"))" + Environment.NewLine +
" allData <- rbind(sa$S, sa$T)" + Environment.NewLine +
" else" + Environment.NewLine +
" allData <- rbind(allData, sa$S, sa$T)" + Environment.NewLine +
"}}" + Environment.NewLine +
"write.table(allData, sep=\",\", row.names=FALSE)" + Environment.NewLine
,
paramNames, NumPaths, Parameters.Count,
sobolx1FileName.Replace("\\", "/"),
sobolx1FileName.Replace("\\", "/"),
sobolVariableValuesFileName.Replace("\\", "/"));
DataTable resultsForYear = null;
try
{
resultsForYear = RunR(script);
// Put output from R into results table.
if (Years.Count() > 1)
{
results.SetIndex(new object[] { year.ToString() });
}
foreach (DataColumn col in resultsForYear.Columns)
{
if (col.DataType == typeof(string))
{
results.SetValues(col.ColumnName, DataTableUtilities.GetColumnAsStrings(resultsForYear, col.ColumnName));
}
else
{
results.SetValues(col.ColumnName, DataTableUtilities.GetColumnAsDoubles(resultsForYear, col.ColumnName));
}
}
}
catch (Exception err)
{
string msg = err.Message;
if (Years.Count() > 1)
{
msg = "Year " + year + ": " + msg;
}
errorsFromR.Add(msg);
}
}
var resultsRawTable = results.ToTable();
resultsRawTable.TableName = Name + "Statistics";
dataStore.Writer.WriteTable(resultsRawTable);
if (errorsFromR.Count > 0)
{
string msg = StringUtilities.BuildString(errorsFromR.ToArray(), Environment.NewLine);
throw new Exception(msg);
}
}
}
/// <summary>Called by the graph presenter to get a list of all annotations to put on the graph.</summary>
/// <param name="annotations">A list of annotations to add to.</param>
public void GetAnnotationsToPutOnGraph(List <Annotation> annotations)
{
Graph parentGraph = Parent.Parent as Graph;
if (data != null && ColumnName != null && xFieldName != null)
{
string columnName = FindColumn(data);
if (columnName != null && data.Columns.Contains(xFieldName))
{
string[] names = DataTableUtilities.GetColumnAsStrings(data, columnName);
List <object> x;
Type columnType = data.Columns[xFieldName].DataType;
if (columnType == typeof(DateTime))
{
x = DataTableUtilities.GetColumnAsDates(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(int))
{
x = DataTableUtilities.GetColumnAsIntegers(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(double))
{
x = DataTableUtilities.GetColumnAsDoubles(data, xFieldName).Cast <object>().ToList();
}
else
{
throw new Exception($"Error in EventNamesOnGraph {Name}: unknown column type '{columnType.FullName}' in column '{xFieldName}'");
}
if (names.Length == x.Count)
{
var baseColour = Color.LightBlue;
var colourMap = new Dictionary <string, Color>();
int startIndex = -1;
string startName = string.Empty;
for (int i = 0; i < names.Length; i++)
{
if (names[i] != "?" && !string.IsNullOrEmpty(names[i]))
{
if (startIndex == -1)
{
startIndex = i;
startName = names[i];
}
else if (names[i] != startName)
{
// Add a line annotation.
var bar = new LineAnnotation();
if (colourMap.ContainsKey(startName))
{
bar.colour = colourMap[startName];
}
else
{
bar.colour = ColourUtilities.ChangeColorBrightness(baseColour, colourMap.Count * 0.2);
colourMap.Add(startName, bar.colour);
}
bar.type = LineType.Dot;
bar.x1 = x[startIndex];
bar.y1 = double.MinValue;
bar.x2 = x[i - 1];
bar.y2 = double.MaxValue;
bar.InFrontOfSeries = false;
bar.ToolTip = startName;
annotations.Add(bar);
startName = names[i];
startIndex = i;
}
}
}
}
}
}
}
/// <summary>Main run method for performing our post simulation calculations</summary>
/// <param name="dataStore">The data store.</param>
public void Run(IDataStore dataStore)
{
DataTable predictedData = dataStore.Reader.GetData("Report", filter: "SimulationName LIKE '" + Name + "%'", orderBy: "SimulationID");
if (predictedData != null)
{
// Determine how many years we have per simulation
DataView view = new DataView(predictedData);
view.RowFilter = "SimulationName='" + Name + "Simulation1'";
Years = DataTableUtilities.GetColumnAsIntegers(view, "Clock.Today.Year");
// Create a table of all predicted values
DataTable predictedValues = new DataTable();
List <string> descriptiveColumnNames = new List <string>();
List <string> variableNames = new List <string>();
foreach (double year in Years)
{
view.RowFilter = "Clock.Today.Year=" + year;
foreach (DataColumn predictedColumn in view.Table.Columns)
{
if (predictedColumn.DataType == typeof(double))
{
double[] valuesForYear = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
if (valuesForYear.Distinct().Count() == 1)
{
if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
{
descriptiveColumnNames.Add(predictedColumn.ColumnName);
}
}
else
{
DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + year, valuesForYear);
if (!variableNames.Contains(predictedColumn.ColumnName))
{
variableNames.Add(predictedColumn.ColumnName);
}
}
}
}
}
// Run R
DataTable eeDataRaw;
DataTable statsDataRaw;
RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);
// Get ee data from R and store in ee table.
// EE data from R looks like:
// "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
// - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
// - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
// - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
// - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
DataView eeView = new DataView(eeDataRaw);
IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", "Year" });
// Create a path variable.
var pathValues = Enumerable.Range(1, NumPaths).ToArray();
foreach (var parameter in Parameters)
{
foreach (DataColumn column in predictedValues.Columns)
{
eeView.RowFilter = "variable = '" + column.ColumnName + "'";
if (eeView.Count != NumPaths)
{
throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
}
int year = Convert.ToInt32(column.ColumnName.Substring(column.ColumnName.Length - 4));
string variableName = column.ColumnName.Substring(0, column.ColumnName.Length - 4);
eeTableKey.SetIndex(new object[] { parameter.Name, year });
List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
for (int i = 0; i < values.Count; i++)
{
values[i] = Math.Abs(values[i]);
}
var runningMean = MathUtilities.RunningAverage(values);
eeTableKey.SetValues("Path", pathValues);
eeTableKey.SetValues(variableName + ".MuStar", runningMean);
}
}
DataTable eeTable = eeTableKey.ToTable();
eeTable.TableName = Name + "PathAnalysis";
// Get stats data from R and store in MuStar table.
// Stats data coming back from R looks like:
// "mu", "mustar", "sigma", "param","variable"
// -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
// -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
// -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
// 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
// -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
// 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
// 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
// -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
"Parameter", "Year"
});
foreach (DataRow row in statsDataRaw.Rows)
{
string variable = row["variable"].ToString();
int year = Convert.ToInt32(variable.Substring(variable.Length - 4));
variable = variable.Substring(0, variable.Length - 4);
tableKey.SetIndex(new object[] { row["param"], year });
tableKey.Set(variable + ".Mu", row["mu"]);
tableKey.Set(variable + ".MuStar", row["mustar"]);
tableKey.Set(variable + ".Sigma", row["sigma"]);
// Need to bring in the descriptive values.
view.RowFilter = "Clock.Today.Year=" + year;
foreach (var descriptiveColumnName in descriptiveColumnNames)
{
var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
if (values.Distinct().Count() == 1)
{
tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
}
}
}
DataTable muStarTable = tableKey.ToTable();
muStarTable.TableName = Name + "Statistics";
dataStore.Writer.WriteTable(eeTable);
dataStore.Writer.WriteTable(muStarTable);
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
/// <param name="dataStore">The data store.</param>
/// <exception cref="ApsimXException">
/// Could not find model data table: + PredictedTableName
/// or
/// Could not find observed data table: + ObservedTableName
/// </exception>
public void Run(IDataStore dataStore)
{
if (PredictedTableName != null && ObservedTableName != null)
{
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
IEnumerable <string> commonCols = predictedDataNames.Intersect(observedDataNames);
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("O.'@field', ");
}
else
{
query.Append("O.'@field' AS 'Observed.@field', P.'@field' AS 'Predicted.@field', ");
}
query.Replace("@field", s);
}
query.Append("FROM " + ObservedTableName + " O INNER JOIN " + PredictedTableName + " P USING (SimulationID) WHERE O.'@match1' = P.'@match1'");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND O.'@match2' = P.'@match2'");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND O.'@match3' = P.'@match3'");
}
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.Append(" AND P.CheckpointID = " + checkpointID);
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("O.'SimulationID' AS 'Observed.SimulationID', P.'SimulationID' AS 'Predicted.SimulationID'", "O.'SimulationID' AS 'SimulationID'");
query = query.Replace("@match1", FieldNameUsedForMatch);
query = query.Replace("@match2", FieldName2UsedForMatch);
query = query.Replace("@match3", FieldName3UsedForMatch);
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.SimulationID in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName));
}
query.Append(")");
}
DataTable predictedObservedData = dataStore.Reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
foreach (string fieldName in commonCols)
{
string units = dataStore.Reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = dataStore.Reader.GetDataUsingSql("SELECT * FROM " + PredictedTableName);
DataTable observedData = dataStore.Reader.GetDataUsingSql("SELECT * FROM " + ObservedTableName);
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
DataTable statsData = new DataTable();
statsData.Columns.Add("SimulationName", typeof(string));
DataTable simulationData = dataStore.Reader.GetData(this.TableName);
if (simulationData != null)
{
// Add required columns.
var columnNames = new List <string>();
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
statsData.Columns.Add($"{ column.ColumnName}Count", typeof(int));
statsData.Columns.Add($"{ column.ColumnName}Total", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Mean", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Min", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Max", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}StdDev", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile10", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile20", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile30", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile40", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile50", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile60", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile70", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile80", typeof(double));
statsData.Columns.Add($"{ column.ColumnName}Percentile90", typeof(double));
columnNames.Add(column.ColumnName);
}
}
var simulationNames = DataTableUtilities.GetColumnAsStrings(simulationData, "SimulationName").Distinct();
DataView view = new DataView(simulationData);
foreach (var simulationName in simulationNames)
{
view.RowFilter = $"[SimulationName]='{simulationName}'";
var newRow = statsData.NewRow();
foreach (var columnName in columnNames)
{
var values = DataTableUtilities.GetColumnAsDoubles(view, columnName);
newRow["SimulationName"] = simulationName;
newRow[$"{ columnName}Count"] = values.Length;
newRow[$"{ columnName}Total"] = MathUtilities.Sum(values);
newRow[$"{ columnName}Mean"] = MathUtilities.Average(values);
newRow[$"{ columnName}Min"] = MathUtilities.Min(values);
newRow[$"{ columnName}Max"] = MathUtilities.Max(values);
newRow[$"{ columnName}StdDev"] = MathUtilities.SampleStandardDeviation(values);
newRow[$"{ columnName}Percentile10"] = MathUtilities.Percentile(values, 0.1);
newRow[$"{ columnName}Percentile20"] = MathUtilities.Percentile(values, 0.2);
newRow[$"{ columnName}Percentile30"] = MathUtilities.Percentile(values, 0.3);
newRow[$"{ columnName}Percentile40"] = MathUtilities.Percentile(values, 0.4);
newRow[$"{ columnName}Percentile50"] = MathUtilities.Percentile(values, 0.5);
newRow[$"{ columnName}Percentile60"] = MathUtilities.Percentile(values, 0.6);
newRow[$"{ columnName}Percentile70"] = MathUtilities.Percentile(values, 0.7);
newRow[$"{ columnName}Percentile80"] = MathUtilities.Percentile(values, 0.8);
newRow[$"{ columnName}Percentile90"] = MathUtilities.Percentile(values, 0.9);
}
statsData.Rows.Add(newRow);
}
// Write the stats data to the DataStore
statsData.TableName = this.Name;
dataStore.Writer.WriteTable(statsData);
}
}
/// <summary>
/// Save the grid back to the model.
/// </summary>
private void SaveGrid()
{
try
{
this.explorerPresenter.CommandHistory.ModelChanged -= this.OnModelChanged;
// Get the data source of the profile grid.
DataTable data = this.view.ProfileGrid.DataSource;
// Maintain a list of all property changes that we need to make.
List <Commands.ChangeProperty.Property> properties = new List <Commands.ChangeProperty.Property>();
// Loop through all non-readonly properties, get an array of values from the data table
// for the property and then set the property value.
for (int i = 0; i < this.propertiesInGrid.Count; i++)
{
// If this property is NOT readonly then set its value.
if (!this.propertiesInGrid[i].IsReadOnly)
{
// Get an array of values for this property.
Array values;
if (this.propertiesInGrid[i].DataType.GetElementType() == typeof(double))
{
values = DataTableUtilities.GetColumnAsDoubles(data, data.Columns[i].ColumnName);
if (!MathUtilities.ValuesInArray((double[])values))
{
values = null;
}
else
{
values = MathUtilities.RemoveMissingValuesFromBottom((double[])values);
}
}
else
{
values = DataTableUtilities.GetColumnAsStrings(data, data.Columns[i].ColumnName);
values = MathUtilities.RemoveMissingValuesFromBottom((string[])values);
}
// Is the value any different to the former property value?
bool changedValues;
if (this.propertiesInGrid[i].DataType == typeof(double[]))
{
changedValues = !MathUtilities.AreEqual((double[])values, (double[])this.propertiesInGrid[i].Value);
}
else
{
changedValues = !MathUtilities.AreEqual((string[])values, (string[])this.propertiesInGrid[i].Value);
}
if (changedValues)
{
// Store the property change.
Commands.ChangeProperty.Property property =
new Commands.ChangeProperty.Property(this.propertiesInGrid[i].Object, this.propertiesInGrid[i].Name, values);
properties.Add(property);
}
}
}
// If there are property changes pending, then commit the changes in a block.
if (properties.Count > 0)
{
Commands.ChangeProperty command = new Commands.ChangeProperty(properties);
this.explorerPresenter.CommandHistory.Add(command);
}
this.explorerPresenter.CommandHistory.ModelChanged += this.OnModelChanged;
}
catch (Exception e)
{
if (e is System.Reflection.TargetInvocationException)
{
e = (e as System.Reflection.TargetInvocationException).InnerException;
}
this.explorerPresenter.MainPresenter.ShowError(e);
}
}
/// <summary>Get a list of values for a field.</summary>
/// <param name="fieldName">The name of the field.</param>
public double[] GetDoubleValues(string fieldName)
{
return(DataTableUtilities.GetColumnAsDoubles(filter, fieldName));
}
/// <summary>Called by the graph presenter to get a list of all annotations to put on the graph.</summary>
/// <param name="annotations">A list of annotations to add to.</param>
public void GetAnnotationsToPutOnGraph(List <Annotation> annotations)
{
Graph parentGraph = Parent.Parent as Graph;
if (data != null && ColumnName != null && xFieldName != null)
{
string columnName = FindColumn(data);
if (columnName != null && data.Columns.Contains(xFieldName))
{
string[] names = DataTableUtilities.GetColumnAsStrings(data, columnName);
List <object> x;
Type columnType = data.Columns[xFieldName].DataType;
if (columnType == typeof(DateTime))
{
x = DataTableUtilities.GetColumnAsDates(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(int))
{
x = DataTableUtilities.GetColumnAsIntegers(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(double))
{
x = DataTableUtilities.GetColumnAsDoubles(data, xFieldName).Cast <object>().ToList();
}
else
{
throw new Exception($"Error in EventNamesOnGraph {Name}: unknown column type '{columnType.FullName}' in column '{xFieldName}'");
}
if (names.Length == x.Count)
{
var baseColour = Color.LightBlue;
var colourMap = new Dictionary <string, Color>();
int startIndex = -1;
string startName = string.Empty;
for (int i = 0; i < names.Length; i++)
{
if (startIndex == -1)
{
startIndex = i;
startName = names[i];
}
else if (names[i] != startName)
{
if (!string.IsNullOrEmpty(startName))
{
// Add a line annotation.
AddAnnotation(annotations, x, baseColour, colourMap, startIndex, startName, i);
}
startName = names[i];
startIndex = i;
}
}
if (startIndex != -1)
{
AddAnnotation(annotations, x, baseColour, colourMap, startIndex, startName, names.Length);
}
}
}
}
}
/// <summary>
/// Save the grid back to the model.
/// </summary>
private void SaveGrid()
{
this.presenter.CommandHistory.ModelChanged -= this.OnModelChanged;
// Get the data source of the profile grid.
DataTable data = this.xYPairsView.VariablesGrid.DataSource;
// Maintain a list of all property changes that we need to make.
List <Commands.ChangeProperty.Property> properties = new List <Commands.ChangeProperty.Property>();
// add missing data as 0 otherwise it will throw an exception
// could make this work as an entire row, but will stick to X & Y columns for now
/*
* for (int Row = 0; Row != data.Rows.Count; Row++)
* {
* if (data.Rows[Row]["Y"].ToString() == "" && data.Rows[Row]["X"].ToString() != "")
* data.Rows[Row]["Y"] = "0";
* if (data.Rows[Row]["X"].ToString() == "" && data.Rows[Row]["Y"].ToString() != "")
* data.Rows[Row]["X"] = "0";
* if (data.Rows[Row]["Y"].ToString() == "" && data.Rows[Row]["X"].ToString() == "")
* break;
* }
*/
//// Loop through all non-readonly properties, get an array of values from the data table
//// for the property and then set the property value.
for (int i = 0; i < this.propertiesInGrid.Count; i++)
{
// If this property is NOT readonly then set its value.
if (!this.propertiesInGrid[i].IsReadOnly)
{
// Get an array of values for this property.
Array values;
if (this.propertiesInGrid[i].DataType.GetElementType() == typeof(double))
{
values = DataTableUtilities.GetColumnAsDoubles(data, data.Columns[i].ColumnName);
if (!MathUtilities.ValuesInArray((double[])values))
{
values = null;
}
else
{
values = MathUtilities.RemoveMissingValuesFromBottom((double[])values);
}
}
else
{
values = DataTableUtilities.GetColumnAsStrings(data, data.Columns[i].ColumnName);
values = MathUtilities.RemoveMissingValuesFromBottom((string[])values);
}
// Is the value any different to the former property value?
bool changedValues;
if (this.propertiesInGrid[i].DataType == typeof(double[]))
{
changedValues = !MathUtilities.AreEqual((double[])values, (double[])this.propertiesInGrid[i].Value);
}
else
{
changedValues = !MathUtilities.AreEqual((string[])values, (string[])this.propertiesInGrid[i].Value);
}
if (changedValues)
{
// Store the property change.
Commands.ChangeProperty.Property property =
new Commands.ChangeProperty.Property(this.propertiesInGrid[i].Object, this.propertiesInGrid[i].Name, values);
properties.Add(property);
}
}
}
// If there are property changes pending, then commit the changes in a block.
if (properties.Count > 0)
{
Commands.ChangeProperty command = new Commands.ChangeProperty(properties);
this.presenter.CommandHistory.Add(command);
}
this.presenter.CommandHistory.ModelChanged += this.OnModelChanged;
}
/// <summary>Called by the graph presenter to get a list of all annotations to put on the graph.</summary>
public IEnumerable <IAnnotation> GetAnnotations()
{
List <IAnnotation> annotations = new List <IAnnotation>();
Graph parentGraph = Parent.Parent as Graph;
if (data != null && ColumnName != null && xFieldName != null)
{
string phenologyColumnName = FindPhenologyStageColumn(data);
if (phenologyColumnName != null && data.Columns.Contains(xFieldName))
{
string[] names = DataTableUtilities.GetColumnAsStrings(data, phenologyColumnName);
List <object> x;
Type columnType = data.Columns[xFieldName].DataType;
if (columnType == typeof(DateTime))
{
x = DataTableUtilities.GetColumnAsDates(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(int))
{
x = DataTableUtilities.GetColumnAsIntegers(data, xFieldName).Cast <object>().ToList();
}
else if (columnType == typeof(double))
{
x = DataTableUtilities.GetColumnAsDoubles(data, xFieldName).Cast <object>().ToList();
}
else
{
throw new Exception($"Error in EventNamesOnGraph {Name}: unknown column type '{columnType.FullName}' in column '{xFieldName}'");
}
if (names.Length == x.Count)
{
for (int i = 0; i < names.Length; i++)
{
if (names[i] != "?" && !string.IsNullOrEmpty(names[i]))
{
// Add a line annotation.
LineAnnotation line = new LineAnnotation();
line.colour = Color.Black;
line.type = LineType.Dot;
line.x1 = x[i];
line.y1 = double.MinValue;
line.x2 = x[i];
line.y2 = double.MaxValue;
annotations.Add(line);
// Add a text annotation.
TextAnnotation text = new TextAnnotation();
text.text = names[i];
text.colour = Color.Black;
text.leftAlign = true;
text.x = x[i];
text.y = double.MinValue;
text.textRotation = 270;
annotations.Add(text);
}
}
}
}
}
return(annotations);
}