/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data;
if (dataStore != null)
{
int start = 0;
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
if (ExperimentFilter != null)
{
string filter = "NAME IN " + "(" + StringUtilities.Build(ExperimentFilter.GetSimulationNames(), delimiter: ",", prefix: "'", suffix: "'") + ")";
data = dataStore.GetData(tableName: view.TableList.SelectedValue, filter: filter, from: start, count: count);
}
else if (SimulationFilter != null)
{
data = dataStore.GetData(simulationName: SimulationFilter.Name,
tableName: view.TableList.SelectedValue,
from: start, count: count);
}
else
{
data = dataStore.GetData(tableName: view.TableList.SelectedValue,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
}
}
else
{
data = new DataTable();
}
return(data);
}
private void OnCLEMValidate(object sender, EventArgs e)
{
// validation is performed here
// this event fires after Activity and Resource validation so that resources are available to check in the validation.
// commencing is too early as Summary has not been created for reporting.
// some values assigned in commencing will not be checked before processing, but will be caught here
// each ZoneCLEM and Market will call this validation for all children
// CLEM components above ZoneCLEM (e.g. RandomNumberGenerator) needs to validate itself
if (!ZoneCLEM.Validate(this, "", this, Summary))
{
string error = "@i:Invalid parameters in model";
// find IStorageReader of simulation
IModel parentSimulation = FindAncestor <Simulation>();
IStorageReader ds = DataStore.Reader;
if (ds.GetData(simulationName: parentSimulation.Name, tableName: "_Messages") != null)
{
DataRow[] dataRows = ds.GetData(simulationName: parentSimulation.Name, tableName: "_Messages").Select().OrderBy(a => a[7].ToString()).ToArray();
// all all current errors and validation problems to error string.
foreach (DataRow dr in dataRows)
{
error += "\n" + dr[6].ToString();
}
}
throw new ApsimXException(this, error);
}
}
private void OnCLEMValidate(object sender, EventArgs e)
{
// validation is performed here
// this event fires after Activity and Resource validation so that resources are available to check in the validation.
// commencing is too early as Summary has not been created for reporting.
// some values assigned in commencing will not be checked before processing, but will be caught here
// each ZoneCLEM and Market will call this validation for all children
// CLEM components above ZoneCLEM (e.g. RandomNumberGenerator) needs to validate itself
if (!Validate(this, "", this, Summary))
{
string error = "@i:Invalid parameters in model";
// find IStorageReader of simulation
IModel parentSimulation = FindAncestor <Simulation>();
IStorageReader ds = DataStore.Reader;
if (ds.GetData(simulationName: parentSimulation.Name, tableName: "_Messages") != null)
{
DataRow[] dataRows = ds.GetData(simulationName: parentSimulation.Name, tableName: "_Messages").Select().OrderBy(a => a[7].ToString()).ToArray();
// all all current errors and validation problems to error string.
foreach (DataRow dr in dataRows)
{
error += "\n" + dr[6].ToString();
}
}
throw new ApsimXException(this, error);
}
if (Clock.StartDate.Year > 1) // avoid checking if clock not set.
{
if ((int)EcologicalIndicatorsCalculationMonth >= Clock.StartDate.Month)
{
// go back from start month in intervals until
DateTime trackDate = new DateTime(Clock.StartDate.Year, (int)EcologicalIndicatorsCalculationMonth, Clock.StartDate.Day);
while (trackDate.AddMonths(-EcologicalIndicatorsCalculationInterval) >= Clock.Today)
{
trackDate = trackDate.AddMonths(-EcologicalIndicatorsCalculationInterval);
}
EcologicalIndicatorsNextDueDate = trackDate;
}
else
{
EcologicalIndicatorsNextDueDate = new DateTime(Clock.StartDate.Year, (int)EcologicalIndicatorsCalculationMonth, Clock.StartDate.Day);
while (Clock.StartDate > EcologicalIndicatorsNextDueDate)
{
EcologicalIndicatorsNextDueDate = EcologicalIndicatorsNextDueDate.AddMonths(EcologicalIndicatorsCalculationInterval);
}
}
}
}
/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data = null;
if (dataStore != null)
{
try
{
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
data = dataStore.GetData(
tableName: ModelName,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
}
catch (Exception e)
{
this.explorerPresenter.MainPresenter.ShowMessage("Error reading data tables." + Environment.NewLine + e.ToString(), Simulation.ErrorLevel.Error);
}
}
else
{
data = new DataTable();
}
return(data);
}
public void ExportDataStoreToEXCEL(object sender, EventArgs e)
{
explorerPresenter.MainPresenter.ShowWaitCursor(true);
List <DataTable> tables = new List <DataTable>();
foreach (string tableName in storage.TableNames)
{
using (DataTable table = storage.GetData(tableName))
{
table.TableName = tableName;
tables.Add(table);
}
}
try
{
string fileName = Path.ChangeExtension(storage.FileName, ".xlsx");
Utility.Excel.WriteToEXCEL(tables.ToArray(), fileName);
explorerPresenter.MainPresenter.ShowMessage("Excel successfully created: " + fileName, Simulation.MessageType.Information);
}
catch (Exception err)
{
explorerPresenter.MainPresenter.ShowError(err);
}
finally
{
explorerPresenter.MainPresenter.ShowWaitCursor(false);
}
}
/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteDataInTable(this.Name);
DataTable statsData = new DataTable();
statsData.Columns.Add("SimulationName", typeof(string));
statsData.Columns.Add("VariableName", typeof(string));
statsData.Columns.Add("n", typeof(string));
statsData.Columns.Add("residual", typeof(double));
statsData.Columns.Add("R^2", typeof(double));
statsData.Columns.Add("RMSD", typeof(double));
statsData.Columns.Add("%", typeof(double));
statsData.Columns.Add("MSD", typeof(double));
statsData.Columns.Add("SB", typeof(double));
statsData.Columns.Add("SDSD", typeof(double));
statsData.Columns.Add("LCS", typeof(double));
DataTable simulationData = dataStore.GetData(this.TableName);
if (simulationData != null)
{
DataView view = new DataView(simulationData);
string[] columnNames = DataTableUtilities.GetColumnNames(simulationData);
foreach (string observedColumnName in columnNames)
{
if (observedColumnName.StartsWith("Observed."))
{
string predictedColumnName = observedColumnName.Replace("Observed.", "Predicted.");
if (simulationData.Columns.Contains(predictedColumnName))
{
DataColumn predictedColumn = simulationData.Columns[predictedColumnName];
DataColumn observedColumn = simulationData.Columns[observedColumnName];
if (predictedColumn.DataType == typeof(double) &&
observedColumn.DataType == typeof(double))
{
// Calculate stats for each simulation and store them in a rows in our stats table.
string[] simulationNames = dataStore.SimulationNames;
foreach (string simulationName in simulationNames)
{
string seriesName = simulationName;
view.RowFilter = "SimulationName = '" + simulationName + "'";
CalcStatsRow(view, observedColumnName, predictedColumnName, seriesName, statsData);
}
// Calculate stats for all simulations and store in a row of the stats table.
string overallSeriesName = "Combined " + observedColumnName.Replace("Observed.", "");
view.RowFilter = null;
CalcStatsRow(view, observedColumnName, predictedColumnName, overallSeriesName, statsData);
}
}
}
}
// Write the stats data to the DataStore
statsData.TableName = this.Name;
dataStore.WriteTable(statsData);
}
}
/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data = null;
if (dataStore != null)
{
try
{
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
data = dataStore.GetData(
tableName: ModelName,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
}
catch (Exception e)
{
this.explorerPresenter.MainPresenter.ShowError(e);
}
}
else
{
data = new DataTable();
}
return(data);
}
/// <summary>
/// Create series definitions assuming the vary by fields are text fields in the table.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="varyByFieldNames">The vary by fields.</param>
/// <param name="whereClauseForInScopeData">An SQL WHERE clause for rows that are in scope.</param>
private List <SeriesDefinition> CreateDefinitionsFromFieldInTable(IStorageReader reader, List <string> varyByFieldNames, string whereClauseForInScopeData)
{
List <SeriesDefinition> definitions = new List <SeriesDefinition>();
var fieldsThatExist = reader.ColumnNames(TableName);
var varyByThatExistInTable = varyByFieldNames.Where(v => fieldsThatExist.Contains(v)).ToList();
var validValuesForEachVaryByField = new List <List <string> >();
foreach (var varyByFieldName in varyByThatExistInTable)
{
var data = reader.GetData(TableName,
fieldNames: new string[] { varyByFieldName },
filter: whereClauseForInScopeData,
distinct: true);
var values = DataTableUtilities.GetColumnAsStrings(data, varyByFieldName).Distinct().ToList();
validValuesForEachVaryByField.Add(values);
}
foreach (var combination in MathUtilities.AllCombinationsOf(validValuesForEachVaryByField.ToArray(), reverse:true))
{
var descriptors = new List <SimulationDescription.Descriptor>();
for (int i = 0; i < combination.Count; i++)
{
descriptors.Add(new SimulationDescription.Descriptor(varyByThatExistInTable[i],
combination[i]));
}
definitions.Add(new SeriesDefinition(this, whereClauseForInScopeData, Filter, descriptors));
}
return(definitions);
}
/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data = null;
if (dataStore != null)
{
try
{
int start = 0;
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
if (ExperimentFilter != null)
{
string filter = "S.NAME IN " + "(" + StringUtilities.Build(ExperimentFilter.GetSimulationNames(), delimiter: ",", prefix: "'", suffix: "'") + ")";
if (!string.IsNullOrEmpty(view.RowFilter.Value))
{
filter += " AND " + view.RowFilter.Value;
}
data = dataStore.GetData(tableName: view.TableList.SelectedValue, filter: filter, from: start, count: count);
}
else if (SimulationFilter != null)
{
data = dataStore.GetData(
simulationName: SimulationFilter.Name,
tableName: view.TableList.SelectedValue,
from: start,
count: count);
}
else
{
data = dataStore.GetData(
tableName: view.TableList.SelectedValue,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
}
}
catch (Exception e)
{
this.explorerPresenter.MainPresenter.ShowError(new Exception("Error reading data tables.", e));
}
}
else
{
data = new DataTable();
}
return(data);
}
/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data = null;
if (dataStore != null)
{
try
{
int start = 0;
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
if (ExperimentFilter != null)
{
string filter = "NAME IN " + "(" + StringUtilities.Build(ExperimentFilter.GetSimulationNames(), delimiter: ",", prefix: "'", suffix: "'") + ")";
data = dataStore.GetData(tableName: view.TableList.SelectedValue, filter: filter, from: start, count: count);
}
else if (SimulationFilter != null)
{
data = dataStore.GetData(
simulationName: SimulationFilter.Name,
tableName: view.TableList.SelectedValue,
from: start,
count: count);
}
else
{
data = dataStore.GetData(
tableName: view.TableList.SelectedValue,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
}
}
catch (Exception e)
{
this.explorerPresenter.MainPresenter.ShowMessage("Error reading data tables." + Environment.NewLine + e.ToString(), Simulation.ErrorLevel.Error);
}
}
else
{
data = new DataTable();
}
return(data);
}
/// <summary>
/// Create a data view from the specified table and filter.
/// </summary>
/// <param name="simulationZones">The list of simulation / zone pairs.</param>
/// <param name="storage">Storage service</param>
private DataTable GetBaseData(IStorageReader storage, List <SimulationZone> simulationZones)
{
// Get a list of all simulation names in all simulationZones.
List <string> simulationNames = new List <string>();
simulationZones.ForEach(sim => simulationNames.AddRange(sim.simulationNames));
string filter = null;
foreach (string simulationName in simulationNames.Distinct())
{
if (filter != null)
{
filter += ",";
}
filter += "'" + simulationName + "'";
}
filter = "SimulationName in (" + filter + ")";
if (Filter != string.Empty)
{
filter = AddToFilter(filter, Filter);
}
List <string> fieldNames = new List <string>();
if (storage.ColumnNames(TableName).Contains("Zone"))
{
fieldNames.Add("Zone");
}
if (XFieldName != null && !XFieldName.Equals("SimulationName"))
{
fieldNames.Add(XFieldName);
}
if (YFieldName != null && !fieldNames.Contains(YFieldName))
{
fieldNames.Add(YFieldName);
}
if (X2FieldName != null && !fieldNames.Contains(X2FieldName))
{
fieldNames.Add(X2FieldName);
}
if (Y2FieldName != null && !fieldNames.Contains(Y2FieldName))
{
fieldNames.Add(Y2FieldName);
}
// Add in column names from annotation series.
foreach (EventNamesOnGraph annotation in Apsim.Children(this, typeof(EventNamesOnGraph)))
{
fieldNames.Add(annotation.ColumnName);
}
return(storage.GetData(tableName: TableName, fieldNames: fieldNames, filter: filter));
}
/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteTable(this.Name);
DataTable simulationData = dataStore.GetData("*", this.TableName);
if (simulationData != null)
{
// Add all the necessary columns to our data table.
DataTable probabilityData = new DataTable();
probabilityData.Columns.Add("Probability", typeof(double));
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
probabilityData.Columns.Add(column.ColumnName, typeof(double));
}
}
string[] simulationNames = dataStore.SimulationNames;
DataView view = new DataView(simulationData);
foreach (string simulationName in simulationNames)
{
view.RowFilter = "SimulationName = '" + simulationName + "'";
int startRow = probabilityData.Rows.Count;
// Add in a simulation column.
string[] simulationNameColumnValues = StringUtilities.CreateStringArray(simulationName, view.Count);
DataTableUtilities.AddColumn(probabilityData, "SimulationName", simulationNameColumnValues, startRow, simulationNameColumnValues.Length);
// Add in the probability column
double[] probabilityValues = MathUtilities.ProbabilityDistribution(view.Count, this.Exceedence);
DataTableUtilities.AddColumn(probabilityData, "Probability", probabilityValues, startRow, view.Count);
// Add in all other numeric columns.
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
double[] values = DataTableUtilities.GetColumnAsDoubles(view, column.ColumnName);
Array.Sort <double>(values);
DataTableUtilities.AddColumn(probabilityData, column.ColumnName, values, startRow, values.Length);
}
}
}
// Write the stats data to the DataStore
probabilityData.TableName = this.Name;
dataStore.WriteTableRaw(probabilityData);
}
}
/// <summary>
/// Create a data view from the specified table and filter.
/// </summary>
/// <param name="factors">The list of simulation / zone pairs.</param>
/// <param name="storage">Storage service</param>
private DataTable GetBaseData(IStorageReader storage, List <ISimulationGeneratorFactors> factors)
{
List <string> fieldNames = new List <string>();
foreach (ISimulationGeneratorFactors factor in factors)
{
fieldNames.Add(factor.ColumnName);
}
if (XFieldName != null)
{
fieldNames.Add(XFieldName);
}
if (YFieldName != null)
{
fieldNames.Add(YFieldName);
}
if (YFieldName != null)
{
if (storage.ColumnNames(TableName).Contains(YFieldName + "Error"))
{
fieldNames.Add(YFieldName + "Error");
}
}
if (X2FieldName != null)
{
fieldNames.Add(X2FieldName);
}
if (Y2FieldName != null)
{
fieldNames.Add(Y2FieldName);
}
// Add in column names from annotation series.
foreach (EventNamesOnGraph annotation in Apsim.Children(this, typeof(EventNamesOnGraph)))
{
fieldNames.Add(annotation.ColumnName);
}
string filterToUse;
if (Filter == null || Filter == string.Empty)
{
filterToUse = CreateRowFilter(factors);
}
else
{
filterToUse = Filter + " AND (" + CreateRowFilter(factors) + ")";
}
return(storage.GetData(tableName: TableName, checkpointName: Checkpoint, fieldNames: fieldNames.Distinct(), filter: filterToUse));
}
/// <summary>Create a text report from tables in this data store.</summary>
/// <param name="storage">The data store.</param>
/// <param name="fileName">Name of the file.</param>
public static void WriteAllTables(IStorageReader storage, string fileName)
{
// Write out each table for this simulation.
foreach (string tableName in storage.TableNames)
{
DataTable data = storage.GetData(tableName);
if (data != null && data.Rows.Count > 0)
{
StreamWriter report = new StreamWriter(Path.ChangeExtension(fileName, "." + tableName + ".csv"));
DataTableUtilities.DataTableToText(data, 0, ",", true, report);
report.Close();
}
}
}
/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteDataInTable(this.Name);
DataTable simulationData = dataStore.GetData(TableName, fieldNames: dataStore.GetTableColumns(TableName));
if (simulationData != null)
{
IndexedDataTable simData = new IndexedDataTable(simulationData, new string[] { FieldToSplitOn });
IndexedDataTable probabilityData = new IndexedDataTable(new string[] { FieldToSplitOn });
foreach (var group in simData.Groups())
{
object keyValue = group.IndexValues[0];
// Add in our key column
probabilityData.SetIndex(new object[] { keyValue });
probabilityData.Set <object>(FieldToSplitOn, keyValue);
// Add in all other numeric columns.
bool haveWrittenProbabilityColumn = false;
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
var values = group.Get <double>(column.ColumnName).ToList();
values.Sort();
if (!haveWrittenProbabilityColumn)
{
// Add in the probability column
double[] probabilityValues = MathUtilities.ProbabilityDistribution(values.Count, this.Exceedence);
probabilityData.SetValues("Probability", probabilityValues);
haveWrittenProbabilityColumn = true;
}
probabilityData.SetValues(column.ColumnName, values);
}
}
}
// Write the stats data to the DataStore
DataTable t = probabilityData.ToTable();
t.TableName = this.Name;
dataStore.WriteTable(t);
}
}
public void ExportDataStoreToEXCEL(object sender, EventArgs e)
{
Cursor.Current = Cursors.WaitCursor;
List <DataTable> tables = new List <DataTable>();
foreach (string tableName in storage.TableNames)
{
DataTable table = storage.GetData(tableName);
table.TableName = tableName;
tables.Add(table);
}
string fileName = Path.ChangeExtension(storage.FileName, ".xlsx");
Utility.Excel.WriteToEXCEL(tables.ToArray(), fileName);
explorerPresenter.MainPresenter.ShowMessage("Excel successfully created: " + fileName, Simulation.ErrorLevel.Information);
Cursor.Current = Cursors.Default;
}
/// <summary>Main run method for performing our post simulation calculations</summary>
/// <param name="dataStore">The data store.</param>
public void Run(IStorageReader dataStore)
{
DataTable predictedData = dataStore.GetData("Report");
if (predictedData != null)
{
DataTable elementalEffects = CreateElementalEffectsTable(predictedData);
dataStore.DeleteDataInTable(elementalEffects.TableName);
dataStore.WriteTable(elementalEffects);
DataTable muStarByPath = CreateMuStarByPath(elementalEffects);
dataStore.DeleteDataInTable(muStarByPath.TableName);
dataStore.WriteTable(muStarByPath);
DataTable muStar = CreateMuStar(muStarByPath);
dataStore.DeleteDataInTable(muStar.TableName);
dataStore.WriteTable(muStar);
}
}
private void OnCLEMValidate(object sender, EventArgs e)
{
// validation is performed here
// this event fires after Activity and Resource validation so that resources are available to check in the validation.
// commencing is too early as Summary has not been created for reporting.
// some values assigned in commencing will not be checked before processing, but will be caught here
if (!Validate(Simulation, ""))
{
string error = "@i:Invalid parameters in model";
// find IStorageReader of simulation
IModel parentSimulation = Apsim.Parent(this, typeof(Simulation));
IStorageReader ds = DataStore.Reader;
DataRow[] dataRows = ds.GetData(simulationName: parentSimulation.Name, tableName: "_Messages").Select().OrderBy(a => a[7].ToString()).ToArray();
// all all current errors and validation problems to error string.
foreach (DataRow dr in dataRows)
{
error += "\n" + dr[6].ToString();
}
throw new ApsimXException(this, error);
}
if (EcologicalIndicatorsCalculationMonth >= Clock.StartDate.Month)
{
// go back from start month in intervals until
DateTime trackDate = new DateTime(Clock.StartDate.Year, EcologicalIndicatorsCalculationMonth, Clock.StartDate.Day);
while (trackDate.AddMonths(-EcologicalIndicatorsCalculationInterval) >= Clock.Today)
{
trackDate = trackDate.AddMonths(-EcologicalIndicatorsCalculationInterval);
}
EcologicalIndicatorsNextDueDate = trackDate;
}
else
{
EcologicalIndicatorsNextDueDate = new DateTime(Clock.StartDate.Year, EcologicalIndicatorsCalculationMonth, Clock.StartDate.Day);
while (Clock.StartDate > EcologicalIndicatorsNextDueDate)
{
EcologicalIndicatorsNextDueDate = EcologicalIndicatorsNextDueDate.AddMonths(EcologicalIndicatorsCalculationInterval);
}
}
}
/// <summary>Get data to show in grid.</summary>
/// <returns>A data table of all data.</returns>
private DataTable GetData()
{
DataTable data = null;
if (dataStore != null)
{
try
{
int count = Utility.Configuration.Settings.MaximumRowsOnReportGrid;
data = dataStore.GetData(
tableName: ModelName,
count: Utility.Configuration.Settings.MaximumRowsOnReportGrid);
if (data != null)
{
// need to filter by current simulation
var filteredData = data.AsEnumerable()
.Where(row => row.Field <String>("SimulationName") == this.SimulationName);
if (filteredData.Any())
{
data = filteredData.CopyToDataTable();
}
}
}
catch (Exception e)
{
this.explorerPresenter.MainPresenter.ShowError(e);
}
}
else
{
data = new DataTable();
}
return(data);
}
/// <summary>
/// Create a message table ready for writing.
/// </summary>
/// <param name="storage">The data store</param>
/// <param name="simulationName">The simulation name to get messages for</param>
/// <returns>The filled message table</returns>
private static DataTable GetMessageTable(IStorageReader storage, string simulationName)
{
DataTable messageTable = new DataTable();
DataTable messages = storage.GetData(simulationName: simulationName, tableName: "_Messages");
if (messages != null && messages.Rows.Count > 0)
{
messageTable.Columns.Add("Date", typeof(string));
messageTable.Columns.Add("Message", typeof(string));
string previousCol1Text = null;
string previousMessage = null;
foreach (DataRow row in messages.Rows)
{
// Work out the column 1 text.
string modelName = (string)row["ComponentName"];
string col1Text;
if (row["Date"].GetType() == typeof(DateTime))
{
DateTime date = (DateTime)row["Date"];
col1Text = date.ToString("yyyy-MM-dd") + " " + modelName;
}
else
{
col1Text = row["Date"].ToString();
}
// If the date and model name have changed then write a row.
if (col1Text != previousCol1Text)
{
if (previousCol1Text != null)
{
messageTable.Rows.Add(new object[] { previousCol1Text, previousMessage });
}
previousMessage = string.Empty;
previousCol1Text = col1Text;
}
else
{
col1Text = null;
}
string message = (string)row["Message"];
Simulation.ErrorLevel errorLevel = (Simulation.ErrorLevel)Enum.Parse(typeof(Simulation.ErrorLevel), row["MessageType"].ToString());
if (errorLevel == Simulation.ErrorLevel.Error)
{
previousMessage += "FATAL ERROR: " + message;
}
else if (errorLevel == Simulation.ErrorLevel.Warning)
{
previousMessage += "WARNING: " + message;
}
else
{
previousMessage += message;
}
previousMessage += "\r\n";
}
if (previousMessage != null)
{
messageTable.Rows.Add(new object[] { previousCol1Text, previousMessage });
}
}
return(messageTable);
}
/// <summary>
/// Write the summary report to the specified writer.
/// </summary>
/// <param name="storage">The data store to query</param>
/// <param name="simulationName">The simulation name to produce a summary report for</param>
/// <param name="writer">Text writer to write to</param>
/// <param name="apsimSummaryImageFileName">The file name for the logo. Can be null</param>
/// <param name="outtype">Indicates the format to be produced</param>
public static void WriteReport(
IStorageReader storage,
string simulationName,
TextWriter writer,
string apsimSummaryImageFileName,
OutputType outtype)
{
Document document = null;
RtfDocumentRenderer renderer = null;
if (outtype == OutputType.html)
{
writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
writer.WriteLine("<html>");
writer.WriteLine("<head>");
writer.WriteLine("<meta content='text/html; charset=UTF-8; http-equiv='content-type'>");
writer.WriteLine("<style>");
writer.WriteLine("h2 { color:darkblue; } ");
writer.WriteLine("h3 { color:darkblue; } ");
writer.WriteLine("table { border:1px solid black; border-collapse:collapse; width:100%; table-layout:fixed; text-align:left; }");
writer.WriteLine("table.headered {text-align:right; }");
writer.WriteLine("tr.total { color:darkorange; font-weight:bold; }");
writer.WriteLine("table.headered td.col1 { text-align:left; font-weight:bold; }");
writer.WriteLine("td { border:1px solid; }");
writer.WriteLine("th { border:1px solid; text-align:right; background-color: palegoldenrod}");
writer.WriteLine("th.col1 { text-align:left; }");
writer.WriteLine("</style>");
writer.WriteLine("</head>");
writer.WriteLine("<body>");
writer.WriteLine("<a href=\"#log\">Simulation log</a>");
}
else if (outtype == OutputType.rtf)
{
document = new Document();
renderer = new RtfDocumentRenderer();
// Get the predefined style Normal.
Style style = document.Styles["Normal"];
// Because all styles are derived from Normal, the next line changes the
// font of the whole document. Or, more exactly, it changes the font of
// all styles and paragraphs that do not redefine the font.
style.Font.Name = "Arial";
// Heading1 to Heading9 are predefined styles with an outline level. An outline level
// other than OutlineLevel.BodyText automatically creates the outline (or bookmarks)
// in PDF.
style = document.Styles["Heading2"];
style.Font.Size = 14;
style.Font.Bold = true;
style.Font.Color = Colors.DarkBlue;
style.ParagraphFormat.PageBreakBefore = false;
style.ParagraphFormat.SpaceAfter = 3;
style.ParagraphFormat.SpaceBefore = 16;
style = document.Styles["Heading3"];
style.Font.Size = 12;
style.Font.Bold = true;
style.Font.Color = Colors.DarkBlue;
style.ParagraphFormat.SpaceBefore = 10;
style.ParagraphFormat.SpaceAfter = 2;
// Create a new style called Monospace based on style Normal
style = document.Styles.AddStyle("Monospace", "Normal");
System.Drawing.FontFamily monoFamily = new System.Drawing.FontFamily(System.Drawing.Text.GenericFontFamilies.Monospace);
style.Font.Name = monoFamily.Name;
Section section = document.AddSection();
}
// Get the initial conditions table.
DataTable initialConditionsTable = storage.GetData(simulationName: simulationName, tableName: "_InitialConditions");
if (initialConditionsTable != null)
{
// Convert the '_InitialConditions' table in the DataStore to a series of
// DataTables for each model.
List <DataTable> tables = new List <DataTable>();
ConvertInitialConditionsToTables(initialConditionsTable, tables);
// Now write all tables to our report.
for (int i = 0; i < tables.Count; i += 2)
{
// Only write something to the summary file if we have something to write.
if (tables[i].Rows.Count > 0 || tables[i + 1].Rows.Count > 0)
{
string heading = tables[i].TableName;
WriteHeading(writer, heading, outtype, document);
// Write the manager script.
if (tables[i].Rows.Count == 1 && tables[i].Rows[0][0].ToString() == "Script code: ")
{
WriteScript(writer, tables[i].Rows[0], outtype, document);
}
else
{
// Write the properties table if we have any properties.
if (tables[i].Rows.Count > 0)
{
WriteTable(writer, tables[i], outtype, "PropertyTable", document);
}
// Write the general data table if we have any data.
if (tables[i + 1].Rows.Count > 0)
{
WriteTable(writer, tables[i + 1], outtype, "ApsimTable", document);
}
}
if (outtype == OutputType.html)
{
writer.WriteLine("<br/>");
}
}
}
}
// Write out all messages.
WriteHeading(writer, "Simulation log:", outtype, document, "log");
DataTable messageTable = GetMessageTable(storage, simulationName);
WriteMessageTable(writer, messageTable, outtype, false, "MessageTable", document);
if (outtype == OutputType.html)
{
writer.WriteLine("</body>");
writer.WriteLine("</html>");
}
else if (outtype == OutputType.rtf)
{
string rtf = renderer.RenderToString(document, Path.GetTempPath());
writer.Write(rtf);
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
if (PredictedTableName == null || ObservedTableName == null)
{
return;
}
// If neither the predicted nor obseved tables have been modified during
// the most recent simulations run, don't do anything.
if (dataStore?.Writer != null &&
!(dataStore.Writer.TablesModified.Contains(PredictedTableName) || dataStore.Writer.TablesModified.Contains(ObservedTableName)))
{
return;
}
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
List <string> commonCols = predictedDataNames.Intersect(observedDataNames).ToList();
// This should be all columns which exist in one table but not both.
IEnumerable <string> uncommonCols = predictedDataNames.Except(observedDataNames).Union(observedDataNames.Except(predictedDataNames));
IStorageReader reader = dataStore.Reader;
string match1ObsShort = reader.BriefColumnName(ObservedTableName, FieldNameUsedForMatch);
string match2ObsShort = reader.BriefColumnName(ObservedTableName, FieldName2UsedForMatch);
string match3ObsShort = reader.BriefColumnName(ObservedTableName, FieldName3UsedForMatch);
string match1PredShort = reader.BriefColumnName(PredictedTableName, FieldNameUsedForMatch);
string match2PredShort = reader.BriefColumnName(PredictedTableName, FieldName2UsedForMatch);
string match3PredShort = reader.BriefColumnName(PredictedTableName, FieldName3UsedForMatch);
StringBuilder query = new StringBuilder("SELECT ");
for (int i = 0; i < commonCols.Count; i++)
{
string s = commonCols[i];
string obsColShort = reader.BriefColumnName(ObservedTableName, s);
string predColShort = reader.BriefColumnName(PredictedTableName, s);
if (i != 0)
{
query.Append(", ");
}
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append($"O.\"{obsColShort}\"");
}
else
{
query.Append($"O.\"{obsColShort}\" AS \"Observed.{obsColShort}\", P.\"{predColShort}\" AS \"Predicted.{predColShort}\"");
}
}
// Add columns which exist in one table but not both.
foreach (string uncommonCol in uncommonCols)
{
// Basically this hack is here to allow error data to be added to the p/o graphs.
// This is kind of terrible, but I really don't want to duplicate every
// column from both predicted and observed tables if we don't have to.
// This does raise the question of whether we should be creating a "PredictedObserved"
// table at all, since we're actually duplicating data in the DB by doing so.
if (AllColumns || uncommonCol.EndsWith("Error"))
{
if (predictedDataNames.Contains(uncommonCol))
{
query.Append($", P.\"{uncommonCol}\" as \"Predicted.{uncommonCol}\"");
}
else if (observedDataNames.Contains(uncommonCol))
{
query.Append($", O.\"{uncommonCol}\" as \"Observed.{uncommonCol}\"");
}
}
}
query.AppendLine();
query.AppendLine($"FROM [{ObservedTableName}] O");
query.AppendLine($"INNER JOIN [{PredictedTableName}] P");
query.Append($"USING ([SimulationID], [CheckpointID], [{FieldNameUsedForMatch}]");
if (!string.IsNullOrEmpty(FieldName2UsedForMatch))
{
query.Append($", \"{FieldName2UsedForMatch}\"");
}
if (!string.IsNullOrEmpty(FieldName3UsedForMatch))
{
query.Append($", \"{FieldName3UsedForMatch}\"");
}
query.AppendLine(")");
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.AppendLine("WHERE [CheckpointID] = " + checkpointID);
query.Replace("O.\"SimulationID\" AS \"Observed.SimulationID\", P.\"SimulationID\" AS \"Predicted.SimulationID\"", "O.\"SimulationID\" AS \"SimulationID\"");
query.Replace("O.\"CheckpointID\" AS \"Observed.CheckpointID\", P.\"CheckpointID\" AS \"Predicted.CheckpointID\"", "O.\"CheckpointID\" AS \"CheckpointID\"");
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in this.FindAllInScope <Experiment>())
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in this.FindAllInScope <Simulation>())
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.[SimulationID] in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName, null));
}
query.Append(")");
}
DataTable predictedObservedData = reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
foreach (DataColumn column in predictedObservedData.Columns)
{
if (column.ColumnName.StartsWith("Predicted."))
{
string shortName = column.ColumnName.Substring("Predicted.".Length);
column.ColumnName = "Predicted." + reader.FullColumnName(PredictedTableName, shortName);
}
else if (column.ColumnName.StartsWith("Observed."))
{
string shortName = column.ColumnName.Substring("Observed.".Length);
column.ColumnName = "Observed." + reader.FullColumnName(ObservedTableName, shortName);
}
else if (column.ColumnName.Equals(match1ObsShort) || column.ColumnName.Equals(match2ObsShort) || column.ColumnName.Equals(match3ObsShort))
{
column.ColumnName = reader.FullColumnName(ObservedTableName, column.ColumnName);
}
}
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
reader.Refresh();
foreach (string fieldName in commonCols)
{
string units = reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
// The Writer replaces tables, rather than appends to them,
// so we actually need to re-write the existing units table values
// Is there a better way to do this?
DataView allUnits = new DataView(reader.GetData("_Units"));
allUnits.Sort = "TableName";
DataTable tableNames = allUnits.ToTable(true, "TableName");
foreach (DataRow row in tableNames.Rows)
{
string tableName = row["TableName"] as string;
List <string> colNames = new List <string>();
List <string> unitz = new List <string>();
foreach (DataRowView rowView in allUnits.FindRows(tableName))
{
colNames.Add(rowView["ColumnHeading"].ToString());
unitz.Add(rowView["Units"].ToString());
}
dataStore.Writer.AddUnits(tableName, colNames, unitz);
}
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = reader.GetDataUsingSql("SELECT * FROM [" + PredictedTableName + "]");
DataTable observedData = reader.GetDataUsingSql("SELECT * FROM [" + ObservedTableName + "]");
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
/// <summary>
/// Run tests
/// </summary>
/// <param name="accept">If true, the stats from this run will be written to file as the accepted stats.</param>
/// <param name="GUIrun">If true, do not raise an exception on test failure.</param>
public void Test(bool accept = false, bool GUIrun = false)
{
PredictedObserved PO = Parent as PredictedObserved;
if (PO == null)
{
return;
}
IStorageReader DS = PO.Parent as IStorageReader;
MathUtilities.RegrStats[] stats;
List <string> statNames = (new MathUtilities.RegrStats()).GetType().GetFields().Select(f => f.Name).ToList(); // use reflection, get names of stats available
DataTable POtable = DS.GetData(PO.Name);
List <string> columnNames;
string sigIdent = "X";
if (POtable == null)
{
object sim = PO.Parent;
while (sim as Simulations == null)
{
sim = ((Model)sim).Parent;
}
throw new ApsimXException(this, "Could not find PO table in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>") + ". Has the simulation been run?");
}
columnNames = POtable.Columns.Cast <DataColumn>().Select(c => c.ColumnName).ToList(); //get list of column names
columnNames = columnNames.Where(c => c.Contains("Observed")).ToList(); //filter names that are not pred/obs pairs
for (int i = 0; i < columnNames.Count; i++)
{
columnNames[i] = columnNames[i].Replace("Observed.", "");
}
columnNames.Sort(); //ensure column names are always in the same order
stats = new MathUtilities.RegrStats[columnNames.Count];
List <double> x = new List <double>();
List <double> y = new List <double>();
string xstr, ystr;
double xres;
double yres;
for (int c = 0; c < columnNames.Count; c++) //on each P/O column pair
{
string observedFieldName = "Observed." + columnNames[c];
string predictedFieldName = "Predicted." + columnNames[c];
if (POtable.Columns.Contains(observedFieldName) &&
POtable.Columns.Contains(predictedFieldName))
{
x.Clear();
y.Clear();
foreach (DataRow row in POtable.Rows)
{
xstr = row[observedFieldName].ToString();
ystr = row[predictedFieldName].ToString();
if (Double.TryParse(xstr, out xres) && Double.TryParse(ystr, out yres))
{
x.Add(xres);
y.Add(yres);
}
}
if (x.Count == 0 || y.Count == 0)
{
continue;
}
stats[c] = MathUtilities.CalcRegressionStats(columnNames[c], y, x);
}
}
//remove any null stats which can occur from non-numeric columns such as dates
List <MathUtilities.RegrStats> list = new List <MathUtilities.RegrStats>(stats);
list.RemoveAll(l => l == null);
stats = list.ToArray();
//remove entries from column names
for (int i = columnNames.Count() - 1; i >= 0; i--)
{
bool found = false;
for (int j = 0; j < stats.Count(); j++)
{
if (columnNames[i] == stats[j].Name)
{
found = true;
break;
}
}
if (!found)
{
columnNames.RemoveAt(i);
}
}
//turn stats array into a DataTable
//first, check if there is already an AcceptedStats array, create if not.
//If the names don't match, then use current stats as user has dragged
//an already existing Test to a new node.
if (AcceptedStats == null || POName != PO.Name)
{
POName = PO.Name;
AcceptedStats = stats;
AcceptedStatsName = StringUtilities.Build(statNames, " ");
}
//then make sure the names and order of the accepted stats are the same as the new ones.
if (StringUtilities.Build(statNames, " ") != AcceptedStatsName)
{
throw new ApsimXException(this, "Names, number or order of accepted stats do not match class MathUtilities.RegrStats. The class has probably changed.");
}
Table = new DataTable("StatTests");
Table.Columns.Add("Name", typeof(string));
Table.Columns.Add("Variable", typeof(string));
Table.Columns.Add("Test", typeof(string));
Table.Columns.Add("Accepted", typeof(double));
Table.Columns.Add("Current", typeof(double));
Table.Columns.Add("Difference", typeof(double));
Table.Columns.Add("Fail?", typeof(string));
double accepted;
double current;
DataTable AcceptedTable = Table.Copy();
DataTable CurrentTable = Table.Copy();
//accepted table
for (int i = 0; i < AcceptedStats.Count(); i++)
{
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
accepted = Convert.ToDouble(AcceptedStats[i].GetType().GetField(statNames[j]).GetValue(AcceptedStats[i]));
AcceptedTable.Rows.Add(PO.Name,
AcceptedStats[i].Name,
statNames[j],
accepted,
null,
null,
null);
}
}
//current table
Table = AcceptedTable.Copy();
int rowIndex = 0;
for (int i = 0; i < stats.Count(); i++)
{
for (int j = 1; j < statNames.Count; j++) //start at 1; we don't want Name field.
{
current = Convert.ToDouble(stats[i].GetType().GetField(statNames[j]).GetValue(stats[i]));
CurrentTable.Rows.Add(PO.Name,
stats[i].Name,
statNames[j],
null,
current,
null,
null);
Table.Rows[rowIndex]["Current"] = current;
rowIndex++;
}
}
//Merge overwrites rows, so add the correct data back in
foreach (DataRow row in Table.Rows)
{
DataRow[] rowAccepted = AcceptedTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
DataRow[] rowCurrent = CurrentTable.Select("Name = '" + row["Name"] + "' AND Variable = '" + row["Variable"] + "' AND Test = '" + row["Test"] + "'");
if (rowAccepted.Count() == 0)
{
row["Accepted"] = DBNull.Value;
}
else
{
row["Accepted"] = rowAccepted[0]["Accepted"];
}
if (rowCurrent.Count() == 0)
{
row["Current"] = DBNull.Value;
}
else
{
row["Current"] = rowCurrent[0]["Current"];
}
if (row["Accepted"] != DBNull.Value && row["Current"] != DBNull.Value)
{
row["Difference"] = Convert.ToDouble(row["Current"]) - Convert.ToDouble(row["Accepted"]);
row["Fail?"] = Math.Abs(Convert.ToDouble(row["Difference"])) > Math.Abs(Convert.ToDouble(row["Accepted"])) * 0.01 ? sigIdent : " ";
}
else
{
row["Difference"] = DBNull.Value;
row["Fail?"] = sigIdent;
}
}
//Tables could be large so free the memory.
AcceptedTable = null;
CurrentTable = null;
if (accept)
{
AcceptedStats = stats;
}
else
{
foreach (DataRow row in Table.Rows)
{
if (row["Fail?"].ToString().Equals(sigIdent))
{
if (!GUIrun)
{
object sim = PO.Parent;
while (sim as Simulations == null)
{
sim = ((Model)sim).Parent;
}
throw new ApsimXException(this, "Significant differences found during regression testing of " + PO.Name + " in " + (sim != null ? ((Simulations)sim).FileName : "<unknown>"));
}
}
}
}
}
/// <summary>
/// Create a datatable that covers a collection of series definitions.
/// </summary>
/// <param name="storage">A data store reader.</param>
/// <param name="series">A list of series definitions.</param>
/// <param name="simulationDescriptions">A list of simulation descriptions.</param>
/// <returns></returns>
private static void ReadAllData(IStorageReader storage, IEnumerable <SeriesDefinition> series,
List <SimulationDescription> simulationDescriptions)
{
var definitionsToProcess = series.ToList();
// Remove all series that already have data. I'm not sure
// under what circumstance this would happen.
definitionsToProcess.RemoveAll(d => d.X != null && d.Y != null);
// Process and remove all series that have a table name specified.
var definitionsWithNoTable = definitionsToProcess.Where(d => d.Series.TableName == null);
foreach (var d in definitionsWithNoTable)
{
d.GetDataFromModels();
}
definitionsToProcess.RemoveAll(d => definitionsWithNoTable.Contains(d));
var allTableNames = definitionsToProcess.Select(d => d.Series.TableName)
.Distinct();
// Get a list of inscope simulation names.
var allSimulationNamesInScope = new List <string>();
foreach (var d in definitionsToProcess)
{
if (d.InScopeSimulationNames != null)
{
allSimulationNamesInScope.AddRange(d.InScopeSimulationNames);
}
}
var inScopeSimulationNames = allSimulationNamesInScope.Distinct();
if (!inScopeSimulationNames.Any())
{
inScopeSimulationNames = null;
}
foreach (var tableName in allTableNames)
{
var definitionsUsingThisTable = definitionsToProcess.Where(d => d.Series.TableName == tableName);
var checkpointNames = definitionsUsingThisTable.Select(d => d.CheckpointName)
.Distinct();
var fieldsThatExist = storage.ColumnNames(tableName);
var fieldNames = definitionsUsingThisTable.SelectMany(d => d.GetFieldNames(fieldsThatExist))
.Distinct();
// Only attempt to read simulation names if this table actually contains
// a simulation name or ID column. Some tables (ie observed data from excel)
// don't necessarily have these columns.
IEnumerable <string> simulationNames = fieldsThatExist.Contains("SimulationID") || fieldsThatExist.Contains("SimulationName") ? inScopeSimulationNames : null;
foreach (var checkpointName in checkpointNames)
{
var table = storage.GetData(tableName, checkpointName, simulationNames, fieldNames);
// Tell each series definition to read its data.
var definitions = definitionsToProcess.Where(d => d.Series.TableName == tableName && d.CheckpointName == checkpointName);
Parallel.ForEach(definitions, (definition) =>
definition.ReadData(table, simulationDescriptions, storage));
}
}
}
/// <summary>Reads all data from the specified reader.</summary>
/// <param name="reader">Storage reader.</param>
/// <param name="simulationDescriptions">Complete list of simulation descriptions.</param>
public void ReadData(IStorageReader reader, List <SimulationDescription> simulationDescriptions)
{
if (X != null && Y != null)
{
return;
}
if (series.TableName == null)
{
if (!String.IsNullOrEmpty(XFieldName))
{
X = GetDataFromModels(XFieldName);
}
if (!String.IsNullOrEmpty(YFieldName))
{
Y = GetDataFromModels(YFieldName);
}
if (!String.IsNullOrEmpty(X2FieldName))
{
X2 = GetDataFromModels(X2FieldName);
}
if (!String.IsNullOrEmpty(Y2FieldName))
{
Y2 = GetDataFromModels(Y2FieldName);
}
}
else
{
var fieldsThatExist = reader.ColumnNames(series.TableName);
// If we have descriptors, then use them to filter the data for this series.
string filter = null;
if (Descriptors != null)
{
foreach (var descriptor in Descriptors)
{
if (fieldsThatExist.Contains(descriptor.Name))
{
filter = AddToFilter(filter, descriptor.Name + " = '" + descriptor.Value + "'");
}
else
{
filter = AddSimulationNameClauseToFilter(filter, descriptor, simulationDescriptions);
}
}
// Incorporate our scope filter if we haven't limited filter to particular simulations.
if (!filter.Contains("SimulationName IN"))
{
filter = AddToFilter(filter, scopeFilter);
}
}
else
{
filter = AddToFilter(filter, scopeFilter);
}
if (!string.IsNullOrEmpty(userFilter))
{
filter = AddToFilter(filter, userFilter);
}
// Get a list of fields to read from data store.
var fieldsToRead = new List <string>();
fieldsToRead.Add(XFieldName);
fieldsToRead.Add(YFieldName);
if (X2FieldName != null)
{
fieldsToRead.Add(X2FieldName);
}
if (Y2FieldName != null)
{
fieldsToRead.Add(Y2FieldName);
}
// Add any error fields to the list of fields to read.
var fieldsToAdd = new List <string>();
foreach (var fieldName in fieldsToRead)
{
if (fieldsThatExist.Contains(fieldName + "Error"))
{
fieldsToAdd.Add(fieldName + "Error");
}
}
fieldsToRead.AddRange(fieldsToAdd);
// Add any field names from the filter.
fieldsToRead.AddRange(ExtractFieldNamesFromFilter(filter));
// Add any fields from child graphable models.
foreach (IGraphable series in Apsim.Children(series, typeof(IGraphable)))
{
fieldsToRead.AddRange(series.GetExtraFieldsToRead(this));
}
// Checkpoints don't exist in observed files so don't pass a checkpoint name to
// GetData in this situation.
string localCheckpointName = CheckpointName;
if (!reader.ColumnNames(series.TableName).Contains("CheckpointID"))
{
localCheckpointName = null;
}
// Go get the data.
Data = reader.GetData(series.TableName, localCheckpointName, fieldNames: fieldsToRead.Distinct(), filter: filter);
// Get the units for our x and y variables.
XFieldUnits = reader.Units(series.TableName, XFieldName);
YFieldUnits = reader.Units(series.TableName, YFieldName);
// If data was found, populate our data (e.g. X and Y) properties.
if (Data.Rows.Count > 0)
{
X = GetDataFromTable(Data, XFieldName);
Y = GetDataFromTable(Data, YFieldName);
X2 = GetDataFromTable(Data, X2FieldName);
Y2 = GetDataFromTable(Data, Y2FieldName);
Error = GetErrorDataFromTable(Data, YFieldName);
if (series.Cumulative)
{
Y = MathUtilities.Cumulative(Y as IEnumerable <double>);
}
if (series.CumulativeX)
{
X = MathUtilities.Cumulative(X as IEnumerable <double>);
}
}
}
}
private string CreateHTML()
{
string htmlString = "<!DOCTYPE html>\n" +
"<html>\n<head>\n<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\" />\n<style>\n" +
"body {font-family: sans-serif, Arial, Helvetica; max-width:1000px; font-size:10pt; }" +
".errorbanner {background-color:red; border-radius:5px 5px 0px 0px; color:white; padding:5px; font-weight:bold }" +
".errorcontent {background-color:#FFFAFA; margin-bottom:20px; border-radius:0px 0px 5px 5px; border-color:red; border-width:1px; border-style:none solid solid solid; padding:10px;}" +
".warningbanner {background-color:orange; border-radius:5px 5px 0px 0px; color:white; padding:5px; font-weight:bold }" +
".warningcontent {background-color:#FFFFFA; margin-bottom:20px; border-radius:0px 0px 5px 5px; border-color:orange; border-width:1px; border-style:none solid solid solid; padding:10px;}" +
".messagebanner {background-color:CornflowerBlue; border-radius:5px 5px 0px 0px; color:white; padding:5px; font-weight:bold }" +
".messagecontent {background-color:#FAFAFF; margin-bottom:20px; border-radius:0px 0px 5px 5px; border-color:CornflowerBlue; border-width:1px; border-style:none solid solid solid; padding:10px;}" +
".okbanner {background-color:green; border-radius:5px 5px 0px 0px; color:white; padding:5px; font-weight:bold }" +
".okcontent {background-color:#FAFFFF; margin-bottom:20px; border-radius:0px 0px 5px 5px; border-color:green; border-width:1px; border-style:none solid solid solid; padding:10px;}" +
".holdermain {margin: 20px 0px 20px 0px}" +
".resourcelink {color:#996633; font-weight:bold; background-color:Cornsilk;border-color:#996633; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px; }" +
".activitylink {color:#009999; font-weight:bold; background-color:floralwhite;border-color:#009999; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px; }" +
".filterlink {border-color:#cc33cc; background-color:#f2e2f2; color:#cc33cc; border-width:1px; border-style:solid; padding: 0px 5px 0px 5px; font-weight:bold; border-radius:3px;}" +
".filelink {color:green; font-weight:bold; background-color:mintcream;border-color:green; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px; }" +
".errorlink {color:white; font-weight:bold; background-color:red;border-color:darkred; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px; }" +
".setvalue {font-weight:bold; background-color:#e8fbfc;border-color:#697c7c; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px;}" +
".otherlink {font-weight:bold; color:#333333; background-color:#eeeeee;border-color:#999999; border-width:1px; border-style:solid; padding:0px 5px 0px 5px; border-radius:3px;}" +
".messageentry {padding:5px 0px 5px 0px; line-height: 1.7em; }" +
".holdermain {margin: 20px 0px 20px 0px}" +
"\n</style>\n</head>\n<body>";
// find IStorageReader of simulation
IModel simulation = Apsim.Parent(model, typeof(Simulation));
IModel simulations = Apsim.Parent(simulation, typeof(Simulations));
IStorageReader ds = Apsim.Children(simulations, typeof(IStorageReader)).FirstOrDefault() as IStorageReader;
DataRow[] dataRows = ds.GetData(simulationName: simulation.Name, tableName: "_Messages").Select().OrderBy(a => a[8].ToString()).ToArray();
foreach (DataRow dr in dataRows)
{
// convert invalid parameter warnings to errors
if (dr[7].ToString().StartsWith("Invalid parameter value in model"))
{
dr[8] = "0";
}
}
dataRows = dataRows.OrderBy(a => a[8].ToString()).ToArray();
if (dataRows.Count() > 0)
{
foreach (DataRow dr in dataRows)
{
bool ignore = false;
string msgStr = dr[7].ToString();
if (msgStr.Contains("@i:"))
{
ignore = true;
}
if (!ignore)
{
string type = "Message";
string title = "Message";
switch (dr[8].ToString())
{
case "0":
type = "Error";
title = "Error";
break;
case "1":
type = "Warning";
title = "Warning";
break;
default:
break;
}
if (type == "Error")
{
if (!msgStr.StartsWith("Invalid parameter value in model"))
{
msgStr = msgStr.Substring(msgStr.IndexOf("\n") + 1);
msgStr = msgStr.Substring(msgStr.IndexOf("\n") + 1);
}
}
if (msgStr.IndexOf(':') >= 0 & msgStr.StartsWith("@"))
{
switch (msgStr.Substring(0, msgStr.IndexOf(':')))
{
case "@error":
type = "Error";
title = "Error";
msgStr = msgStr.Substring(msgStr.IndexOf(':') + 1);
break;
case "@validation":
type = "Error";
title = "Validation error";
msgStr = msgStr.Replace("PARAMETER:", "<b>Parameter:</b>");
msgStr = msgStr.Replace("DESCRIPTION:", "<b>Description:</b>");
msgStr = msgStr.Replace("PROBLEM:", "<b>Problem:</b>");
msgStr = msgStr.Substring(msgStr.IndexOf(':') + 1);
break;
}
}
if (msgStr.Contains("terminated normally"))
{
type = "Ok";
title = "Success";
DataTable dataRows2 = ds.GetData(simulationName: simulation.Name, tableName: "_InitialConditions");
DateTime lastrun = DateTime.Parse(dataRows2.Rows[2][11].ToString());
msgStr = "Simulation successfully completed at [" + lastrun.ToShortTimeString() + "] on [" + lastrun.ToShortDateString() + "]";
}
htmlString += "\n<div class=\"holdermain\">";
htmlString += "\n<div class=\"" + type.ToLower() + "banner\">" + title + "</div>";
htmlString += "\n<div class=\"" + type.ToLower() + "content\">";
msgStr = msgStr.Replace("\n", "<br />");
msgStr = msgStr.Replace("]", "</span>");
msgStr = msgStr.Replace("[r=", "<span class=\"resourcelink\">");
msgStr = msgStr.Replace("[a=", "<span class=\"activitylink\">");
msgStr = msgStr.Replace("[f=", "<span class=\"filterlink\">");
msgStr = msgStr.Replace("[x=", "<span class=\"filelink\">");
msgStr = msgStr.Replace("[o=", "<span class=\"otherlink\">");
msgStr = msgStr.Replace("[", "<span class=\"setvalue\">");
htmlString += "\n<div class=\"messageentry\">" + msgStr;
htmlString += "\n</div>";
htmlString += "\n</div>";
htmlString += "\n</div>";
}
}
}
else
{
htmlString += "\n<div class=\"holdermain\">";
htmlString += "\n <div class=\"messagebanner\">Message</div>";
htmlString += "\n <div class=\"messagecontent\">";
htmlString += "\n <div class=\"activityentry\">This simulation has not been performed";
htmlString += "\n </div>";
htmlString += "\n </div>";
htmlString += "\n</div>";
}
htmlString += "\n</body>\n</html>";
return(htmlString);
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
if (PredictedTableName != null && ObservedTableName != null)
{
IEnumerable <string> predictedDataNames = dataStore.Reader.ColumnNames(PredictedTableName);
IEnumerable <string> observedDataNames = dataStore.Reader.ColumnNames(ObservedTableName);
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + PredictedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
// get the common columns between these lists of columns
IEnumerable <string> commonCols = predictedDataNames.Intersect(observedDataNames);
IStorageReader reader = dataStore.Reader;
string match1ObsShort = reader.BriefColumnName(ObservedTableName, FieldNameUsedForMatch);
string match2ObsShort = reader.BriefColumnName(ObservedTableName, FieldName2UsedForMatch);
string match3ObsShort = reader.BriefColumnName(ObservedTableName, FieldName3UsedForMatch);
string match1PredShort = reader.BriefColumnName(PredictedTableName, FieldNameUsedForMatch);
string match2PredShort = reader.BriefColumnName(PredictedTableName, FieldName2UsedForMatch);
string match3PredShort = reader.BriefColumnName(PredictedTableName, FieldName3UsedForMatch);
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
string obsColShort = reader.BriefColumnName(ObservedTableName, s);
string predColShort = reader.BriefColumnName(PredictedTableName, s);
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("O.[" + obsColShort + "], ");
}
else
{
query.Append("O.[" + obsColShort + "] AS [Observed." + obsColShort + "], P.[" + predColShort + "] AS [Predicted." + predColShort + "], ");
}
}
query.Append("FROM [" + ObservedTableName + "] O INNER JOIN [" + PredictedTableName + "] P USING ([SimulationID]) WHERE O.[" + match1ObsShort + "] = P.[" + match1PredShort + "]");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match2ObsShort + "] = P.[" + match2PredShort + "]");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND O.[" + match3ObsShort + "] = P.[" + match3PredShort + "]");
}
int checkpointID = dataStore.Writer.GetCheckpointID("Current");
query.Append(" AND P.[CheckpointID] = " + checkpointID);
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("O.[SimulationID] AS [Observed.SimulationID], P.[SimulationID] AS [Predicted.SimulationID]", "O.[SimulationID] AS [SimulationID]");
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
var names = experiment.GenerateSimulationDescriptions().Select(s => s.Name);
simulationNames.AddRange(names);
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND O.[SimulationID] in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.Writer.GetSimulationID(simulationName, null));
}
query.Append(")");
}
DataTable predictedObservedData = reader.GetDataUsingSql(query.ToString());
if (predictedObservedData != null)
{
foreach (DataColumn column in predictedObservedData.Columns)
{
if (column.ColumnName.StartsWith("Predicted."))
{
string shortName = column.ColumnName.Substring("Predicted.".Length);
column.ColumnName = "Predicted." + reader.FullColumnName(PredictedTableName, shortName);
}
else if (column.ColumnName.StartsWith("Observed."))
{
string shortName = column.ColumnName.Substring("Observed.".Length);
column.ColumnName = "Observed." + reader.FullColumnName(ObservedTableName, shortName);
}
else if (column.ColumnName.Equals(match1ObsShort) || column.ColumnName.Equals(match2ObsShort) || column.ColumnName.Equals(match3ObsShort))
{
column.ColumnName = reader.FullColumnName(ObservedTableName, column.ColumnName);
}
}
// Add in error columns for each data column.
foreach (string columnName in commonCols)
{
if (predictedObservedData.Columns.Contains("Predicted." + columnName) &&
predictedObservedData.Columns["Predicted." + columnName].DataType == typeof(double))
{
var predicted = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Predicted." + columnName);
var observed = DataTableUtilities.GetColumnAsDoubles(predictedObservedData, "Observed." + columnName);
if (predicted.Length > 0 && predicted.Length == observed.Length)
{
var errorData = MathUtilities.Subtract(predicted, observed);
var errorColumnName = "Pred-Obs." + columnName;
var errorColumn = predictedObservedData.Columns.Add(errorColumnName, typeof(double));
DataTableUtilities.AddColumn(predictedObservedData, errorColumnName, errorData);
predictedObservedData.Columns[errorColumnName].SetOrdinal(predictedObservedData.Columns["Predicted." + columnName].Ordinal + 1);
}
}
}
// Write table to datastore.
predictedObservedData.TableName = this.Name;
dataStore.Writer.WriteTable(predictedObservedData);
List <string> unitFieldNames = new List <string>();
List <string> unitNames = new List <string>();
// write units to table.
reader.Refresh();
foreach (string fieldName in commonCols)
{
string units = reader.Units(PredictedTableName, fieldName);
if (units != null && units != "()")
{
string unitsMinusBrackets = units.Replace("(", "").Replace(")", "");
unitFieldNames.Add("Predicted." + fieldName);
unitNames.Add(unitsMinusBrackets);
unitFieldNames.Add("Observed." + fieldName);
unitNames.Add(unitsMinusBrackets);
}
}
if (unitNames.Count > 0)
{
// The Writer replaces tables, rather than appends to them,
// so we actually need to re-write the existing units table values
// Is there a better way to do this?
DataView allUnits = new DataView(reader.GetData("_Units"));
allUnits.Sort = "TableName";
DataTable tableNames = allUnits.ToTable(true, "TableName");
foreach (DataRow row in tableNames.Rows)
{
string tableName = row["TableName"] as string;
List <string> colNames = new List <string>();
List <string> unitz = new List <string>();
foreach (DataRowView rowView in allUnits.FindRows(tableName))
{
colNames.Add(rowView["ColumnHeading"].ToString());
unitz.Add(rowView["Units"].ToString());
}
dataStore.Writer.AddUnits(tableName, colNames, unitz);
}
dataStore.Writer.AddUnits(Name, unitFieldNames, unitNames);
}
}
else
{
// Determine what went wrong.
DataTable predictedData = reader.GetDataUsingSql("SELECT * FROM [" + PredictedTableName + "]");
DataTable observedData = reader.GetDataUsingSql("SELECT * FROM [" + ObservedTableName + "]");
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
