/// <summary>
/// Format the grid.
/// </summary>
private void FormatGrid()
{
for (int i = 0; i < this.properties.Count; i++)
{
IGridCell cell = this.grid.GetCell(1, i);
if (this.properties[i].DisplayType == DisplayAttribute.DisplayTypeEnum.TableName)
{
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = this.storage.TableNames.ToArray();
}
else if (this.properties[i].DisplayType == DisplayAttribute.DisplayTypeEnum.CultivarName)
{
cell.EditorType = EditorTypeEnum.DropDown;
ICrop crop = this.GetCrop(this.properties);
if (crop != null)
{
cell.DropDownStrings = this.GetCultivarNames(crop);
}
}
else if (this.properties[i].DisplayType == DisplayAttribute.DisplayTypeEnum.FileName)
{
cell.EditorType = EditorTypeEnum.Button;
}
else if (this.properties[i].DisplayType == DisplayAttribute.DisplayTypeEnum.FieldName)
{
cell.EditorType = EditorTypeEnum.DropDown;
string[] fieldNames = this.GetFieldNames();
if (fieldNames != null)
{
cell.DropDownStrings = fieldNames;
}
}
else
{
object cellValue = this.properties[i].ValueWithArrayHandling;
if (cellValue is DateTime)
{
cell.EditorType = EditorTypeEnum.DateTime;
}
else if (cellValue is bool)
{
cell.EditorType = EditorTypeEnum.Boolean;
}
else if (cellValue.GetType().IsEnum)
{
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = StringUtilities.EnumToStrings(cellValue);
}
else if (cellValue.GetType() == typeof(ICrop))
{
cell.EditorType = EditorTypeEnum.DropDown;
List <string> cropNames = new List <string>();
foreach (Model crop in Apsim.FindAll(this.model, typeof(ICrop)))
{
cropNames.Add(crop.Name);
}
cell.DropDownStrings = cropNames.ToArray();
}
else if (this.properties[i].DataType == typeof(ICrop))
{
List <string> plantNames = Apsim.FindAll(this.model, typeof(ICrop)).Select(m => m.Name).ToList();
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = plantNames.ToArray();
}
else
{
cell.EditorType = EditorTypeEnum.TextBox;
}
}
}
IGridColumn descriptionColumn = this.grid.GetColumn(0);
descriptionColumn.Width = -1;
descriptionColumn.ReadOnly = true;
IGridColumn valueColumn = this.grid.GetColumn(1);
valueColumn.Width = -1;
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
/// <param name="dataStore">The data store.</param>
/// <exception cref="ApsimXException">
/// Could not find model data table: + ObservedTableName
/// or
/// Could not find observed data table: + ObservedTableName
/// </exception>
public void Run(IStorageReader dataStore)
{
if (PredictedTableName != null && ObservedTableName != null)
{
dataStore.DeleteDataInTable(this.Name);
DataTable predictedDataNames = dataStore.RunQuery("PRAGMA table_info(" + PredictedTableName + ")");
DataTable observedDataNames = dataStore.RunQuery("PRAGMA table_info(" + ObservedTableName + ")");
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + ObservedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
IEnumerable <string> commonCols = from p in predictedDataNames.AsEnumerable()
join o in observedDataNames.AsEnumerable() on p["name"] equals o["name"]
select p["name"] as string;
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("I.'@field', ");
}
else
{
query.Append("I.'@field' AS 'Observed.@field', R.'@field' AS 'Predicted.@field', ");
}
query.Replace("@field", s);
}
query.Append("FROM " + ObservedTableName + " I INNER JOIN " + PredictedTableName + " R USING (SimulationID) WHERE I.'@match1' = R.'@match1'");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND I.'@match2' = R.'@match2'");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND I.'@match3' = R.'@match3'");
}
int checkpointID = dataStore.GetCheckpointID("Current");
query.Append(" AND R.CheckpointID = " + checkpointID);
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("I.'SimulationID' AS 'Observed.SimulationID', R.'SimulationID' AS 'Predicted.SimulationID'", "I.'SimulationID' AS 'SimulationID'");
query = query.Replace("@match1", FieldNameUsedForMatch);
query = query.Replace("@match2", FieldName2UsedForMatch);
query = query.Replace("@match3", FieldName3UsedForMatch);
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
simulationNames.AddRange(experiment.GetSimulationNames());
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND I.SimulationID in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.GetSimulationID(simulationName));
}
query.Append(")");
}
DataTable predictedObservedData = dataStore.RunQuery(query.ToString());
if (predictedObservedData != null)
{
predictedObservedData.TableName = this.Name;
dataStore.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
foreach (string fieldName in commonCols)
{
string units = dataStore.GetUnits(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
dataStore.AddUnitsForTable(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = dataStore.RunQuery("SELECT * FROM " + PredictedTableName);
DataTable observedData = dataStore.RunQuery("SELECT * FROM " + ObservedTableName);
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
/// <summary>
/// Format the grid. (Just one column)
/// </summary>
private void FormatGrid(int propListIndex)
{
for (int i = 0; i < this.properties[propListIndex].Count; i++)
{
IGridCell cell = this.grid.GetCell(propListIndex + 1, i); //add one because of the Description column
if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.TableName)
{
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = this.storage.Reader.TableNames.ToArray();
}
else if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.CultivarName)
{
cell.EditorType = EditorTypeEnum.DropDown;
IPlant crop = this.GetCrop(this.properties[propListIndex]);
if (crop != null)
{
cell.DropDownStrings = this.GetCultivarNames(crop);
}
}
else if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.FileName)
{
cell.EditorType = EditorTypeEnum.Button;
}
else if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.FieldName)
{
cell.EditorType = EditorTypeEnum.DropDown;
string[] fieldNames = this.GetFieldNames(propListIndex);
if (fieldNames != null)
{
cell.DropDownStrings = fieldNames;
}
}
else if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.ResidueName &&
this.model is SurfaceOrganicMatter)
{
cell.EditorType = EditorTypeEnum.DropDown;
string[] fieldNames = this.GetResidueNames();
if (fieldNames != null)
{
cell.DropDownStrings = fieldNames;
}
}
else if (this.properties[propListIndex][i].Display != null && this.properties[propListIndex][i].Display.Type == DisplayType.CLEMResourceName)
{
cell.EditorType = EditorTypeEnum.DropDown;
List <string> fieldNames = new List <string>();
fieldNames.AddRange(this.GetCLEMResourceNames(this.properties[propListIndex][i].Display.CLEMResourceNameResourceGroups));
// add any extras elements provided to the list.
if (this.properties[propListIndex][i].Display.CLEMExtraEntries != null)
{
fieldNames.AddRange(this.properties[propListIndex][i].Display.CLEMExtraEntries);
}
if (fieldNames.Count != 0)
{
cell.DropDownStrings = fieldNames.ToArray();
}
}
else
{
object cellValue = this.properties[propListIndex][i].ValueWithArrayHandling;
if (cellValue is DateTime)
{
cell.EditorType = EditorTypeEnum.DateTime;
}
else if (cellValue is bool)
{
cell.EditorType = EditorTypeEnum.Boolean;
}
else if (cellValue.GetType().IsEnum)
{
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = StringUtilities.EnumToStrings(cellValue);
}
else if (cellValue.GetType() == typeof(IPlant))
{
cell.EditorType = EditorTypeEnum.DropDown;
List <string> cropNames = new List <string>();
foreach (Model crop in Apsim.FindAll(this.model, typeof(IPlant)))
{
cropNames.Add(crop.Name);
}
cell.DropDownStrings = cropNames.ToArray();
}
else if (this.properties[propListIndex][i].DataType == typeof(IPlant))
{
List <string> plantNames = Apsim.FindAll(this.model, typeof(IPlant)).Select(m => m.Name).ToList();
cell.EditorType = EditorTypeEnum.DropDown;
cell.DropDownStrings = plantNames.ToArray();
}
else
{
cell.EditorType = EditorTypeEnum.TextBox;
}
}
}
IGridColumn descriptionColumn = this.grid.GetColumn(0);
descriptionColumn.Width = -1;
descriptionColumn.ReadOnly = true;
IGridColumn valueColumn = this.grid.GetColumn(1);
valueColumn.Width = -1;
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
/// <param name="dataStore">The data store.</param>
/// <exception cref="ApsimXException">
/// Could not find model data table: + PredictedTableName
/// or
/// Could not find observed data table: + ObservedTableName
/// </exception>
public void Run(IDataStore dataStore)
{
if (PredictedTableName != null && ObservedTableName != null)
{
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
IEnumerable <string> commonCols = predictedDataNames.Intersect(observedDataNames);
IStorageReader reader = dataStore.Reader;
string match1ObsShort = reader.BriefColumnName(ObservedTableName, FieldNameUsedForMatch);
string match2ObsShort = reader.BriefColumnName(ObservedTableName, FieldName2UsedForMatch);
string match3ObsShort = reader.BriefColumnName(ObservedTableName, FieldName3UsedForMatch);
string match1PredShort = reader.BriefColumnName(PredictedTableName, FieldNameUsedForMatch);
string match2PredShort = reader.BriefColumnName(PredictedTableName, FieldName2UsedForMatch);
string match3PredShort = reader.BriefColumnName(PredictedTableName, FieldName3UsedForMatch);
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
string obsColShort = reader.BriefColumnName(ObservedTableName, s);
string predColShort = reader.BriefColumnName(PredictedTableName, s);
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("O.[" + obsColShort + "], ");
}
else
{
query.Append("O.[" + obsColShort + "] AS [Observed." + obsColShort + "], P.[" + predColShort + "] AS [Predicted." + predColShort + "], ");
}
}
query.Append("FROM [" + ObservedTableName + "] O INNER JOIN [" + PredictedTableName + "] P USING ([SimulationID]) WHERE O.[" + match1ObsShort + "] = P.[" + match1PredShort + "]");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match2ObsShort + "] = P.[" + match2PredShort + "]");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match3ObsShort + "] = P.[" + match3PredShort + "]");
}
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.Append(" AND P.[CheckpointID] = " + checkpointID);
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("O.[SimulationID] AS [Observed.SimulationID], P.[SimulationID] AS [Predicted.SimulationID]", "O.[SimulationID] AS [SimulationID]");
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.[SimulationID] in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName, null));
}
query.Append(")");
}
DataTable predictedObservedData = reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
foreach (DataColumn column in predictedObservedData.Columns)
{
if (column.ColumnName.StartsWith("Predicted."))
{
string shortName = column.ColumnName.Substring("Predicted.".Length);
column.ColumnName = "Predicted." + reader.FullColumnName(PredictedTableName, shortName);
}
else if (column.ColumnName.StartsWith("Observed."))
{
string shortName = column.ColumnName.Substring("Observed.".Length);
column.ColumnName = "Observed." + reader.FullColumnName(ObservedTableName, shortName);
}
else if (column.ColumnName.Equals(match1ObsShort) || column.ColumnName.Equals(match2ObsShort) || column.ColumnName.Equals(match3ObsShort))
{
column.ColumnName = reader.FullColumnName(ObservedTableName, column.ColumnName);
}
}
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
reader.Refresh();
foreach (string fieldName in commonCols)
{
string units = reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
// The Writer replaces tables, rather than appends to them,
// so we actually need to re-write the existing units table values
// Is there a better way to do this?
DataView allUnits = new DataView(reader.GetData("_Units"));
allUnits.Sort = "TableName";
DataTable tableNames = allUnits.ToTable(true, "TableName");
foreach (DataRow row in tableNames.Rows)
{
string tableName = row["TableName"] as string;
List <string> colNames = new List <string>();
List <string> unitz = new List <string>();
foreach (DataRowView rowView in allUnits.FindRows(tableName))
{
colNames.Add(rowView["ColumnHeading"].ToString());
unitz.Add(rowView["Units"].ToString());
}
dataStore.Writer.AddUnits(tableName, colNames, unitz);
}
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = reader.GetDataUsingSql("SELECT * FROM [" + PredictedTableName + "]");
DataTable observedData = reader.GetDataUsingSql("SELECT * FROM [" + ObservedTableName + "]");
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
/// <summary>
/// Create an initial conditions table in the DataStore.
/// </summary>
private void CreateInitialConditionsTable()
{
var initConditions = new DataTable("_InitialConditions");
initConditions.Columns.Add("SimulationName", typeof(string));
initConditions.Columns.Add("ModelPath", typeof(string));
initConditions.Columns.Add("Name", typeof(string));
initConditions.Columns.Add("Description", typeof(string));
initConditions.Columns.Add("DataType", typeof(string));
initConditions.Columns.Add("Units", typeof(string));
initConditions.Columns.Add("DisplayFormat", typeof(string));
initConditions.Columns.Add("Total", typeof(int));
initConditions.Columns.Add("Value", typeof(string));
string simulationPath = Apsim.FullPath(simulation);
var row = initConditions.NewRow();
row.ItemArray = new object[] { simulation.Name, simulationPath, "Simulation name", "Simulation name", "String", string.Empty, string.Empty, 0, simulation.Name };
initConditions.Rows.Add(row);
row = initConditions.NewRow();
row.ItemArray = new object[] { simulation.Name, simulationPath, "APSIM version", "APSIM version", "String", string.Empty, string.Empty, 0, simulation.ApsimVersion };
initConditions.Rows.Add(row);
row = initConditions.NewRow();
row.ItemArray = new object[] { simulation.Name, simulationPath, "Run on", "Run on", "String", string.Empty, string.Empty, 0, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") };
initConditions.Rows.Add(row);
// Get all model properties and store in 'initialConditionsTable'
foreach (Model model in Apsim.FindAll(simulation))
{
string thisRelativeModelPath = Apsim.FullPath(model).Replace(simulationPath + ".", string.Empty);
var properties = new List <Tuple <string, VariableProperty> >();
FindAllProperties(model, properties);
foreach (var tuple in properties)
{
string propertyValue = tuple.Item2.ValueAsString();
if (propertyValue != string.Empty)
{
if (propertyValue != null && tuple.Item2.DataType == typeof(DateTime))
{
propertyValue = ((DateTime)tuple.Item2.Value).ToString("yyyy-MM-dd HH:mm:ss");
}
int total;
if (double.IsNaN(tuple.Item2.Total))
{
total = 0;
}
else
{
total = 1;
}
if (tuple.Item2.Units == null)
{
tuple.Item2.Units = string.Empty;
}
row = initConditions.NewRow();
row.ItemArray = new object[] { simulation.Name, thisRelativeModelPath, tuple.Item1, tuple.Item2.Description, tuple.Item2.DataType.Name, tuple.Item2.Units, tuple.Item2.Format, total, propertyValue };
initConditions.Rows.Add(row);
}
}
}
storage.Writer.WriteTable(initConditions);
}
