/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteTable(this.Name);
DataTable statsData = new DataTable();
statsData.Columns.Add("SimulationName", typeof(string));
statsData.Columns.Add("VariableName", typeof(string));
statsData.Columns.Add("n", typeof(string));
statsData.Columns.Add("residual", typeof(double));
statsData.Columns.Add("R^2", typeof(double));
statsData.Columns.Add("RMSD", typeof(double));
statsData.Columns.Add("%", typeof(double));
statsData.Columns.Add("MSD", typeof(double));
statsData.Columns.Add("SB", typeof(double));
statsData.Columns.Add("SDSD", typeof(double));
statsData.Columns.Add("LCS", typeof(double));
DataTable simulationData = dataStore.GetData(this.TableName);
if (simulationData != null)
{
DataView view = new DataView(simulationData);
string[] columnNames = DataTableUtilities.GetColumnNames(simulationData);
foreach (string observedColumnName in columnNames)
{
if (observedColumnName.StartsWith("Observed."))
{
string predictedColumnName = observedColumnName.Replace("Observed.", "Predicted.");
if (simulationData.Columns.Contains(predictedColumnName))
{
DataColumn predictedColumn = simulationData.Columns[predictedColumnName];
DataColumn observedColumn = simulationData.Columns[observedColumnName];
if (predictedColumn.DataType == typeof(double) &&
observedColumn.DataType == typeof(double))
{
// Calculate stats for each simulation and store them in a rows in our stats table.
string[] simulationNames = dataStore.SimulationNames;
foreach (string simulationName in simulationNames)
{
string seriesName = simulationName;
view.RowFilter = "SimulationName = '" + simulationName + "'";
CalcStatsRow(view, observedColumnName, predictedColumnName, seriesName, statsData);
}
// Calculate stats for all simulations and store in a row of the stats table.
string overallSeriesName = "Combined " + observedColumnName.Replace("Observed.", "");
view.RowFilter = null;
CalcStatsRow(view, observedColumnName, predictedColumnName, overallSeriesName, statsData);
}
}
}
}
// Write the stats data to the DataStore
statsData.TableName = this.Name;
dataStore.WriteTableRaw(statsData);
}
}
/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteTable(this.Name);
DataTable simulationData = dataStore.GetData("*", this.TableName);
if (simulationData != null)
{
// Add all the necessary columns to our data table.
DataTable probabilityData = new DataTable();
probabilityData.Columns.Add("Probability", typeof(double));
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
probabilityData.Columns.Add(column.ColumnName, typeof(double));
}
}
string[] simulationNames = dataStore.SimulationNames;
DataView view = new DataView(simulationData);
foreach (string simulationName in simulationNames)
{
view.RowFilter = "SimulationName = '" + simulationName + "'";
int startRow = probabilityData.Rows.Count;
// Add in a simulation column.
string[] simulationNameColumnValues = StringUtilities.CreateStringArray(simulationName, view.Count);
DataTableUtilities.AddColumn(probabilityData, "SimulationName", simulationNameColumnValues, startRow, simulationNameColumnValues.Length);
// Add in the probability column
double[] probabilityValues = MathUtilities.ProbabilityDistribution(view.Count, this.Exceedence);
DataTableUtilities.AddColumn(probabilityData, "Probability", probabilityValues, startRow, view.Count);
// Add in all other numeric columns.
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
double[] values = DataTableUtilities.GetColumnAsDoubles(view, column.ColumnName);
Array.Sort <double>(values);
DataTableUtilities.AddColumn(probabilityData, column.ColumnName, values, startRow, values.Length);
}
}
}
// Write the stats data to the DataStore
probabilityData.TableName = this.Name;
dataStore.WriteTableRaw(probabilityData);
}
}
/// <summary>
/// Main run method for performing our calculations and storing data.
/// </summary>
public void Run(IStorageReader dataStore)
{
string fullFileName = FullFileName;
if (fullFileName != null)
{
Simulations simulations = Apsim.Parent(this, typeof(Simulations)) as Simulations;
dataStore.DeleteTable(Name);
DataTable data = GetTable();
data.TableName = this.Name;
dataStore.WriteTableRaw(data);
}
}
/// <summary>
/// Main run method for performing our calculations and storing data.
/// </summary>
/// <param name="dataStore">The data store to store the data</param>
public void Run(IStorageReader dataStore)
{
string fullFileName = AbsoluteFileName;
if (fullFileName != null && File.Exists(fullFileName))
{
dataStore.DeleteTable(this.Name);
// Open the file
FileStream stream = File.Open(fullFileName, FileMode.Open, FileAccess.Read);
// Create a reader.
IExcelDataReader excelReader;
if (Path.GetExtension(fullFileName).Equals(".xls", StringComparison.CurrentCultureIgnoreCase))
{
throw new Exception("EXCEL file must be in .xlsx format. Filename: " + fullFileName);
}
else
{
// Reading from a OpenXml Excel file (2007 format; *.xlsx)
excelReader = ExcelReaderFactory.CreateOpenXmlReader(stream);
}
// Read all sheets from the EXCEL file as a data set
excelReader.IsFirstRowAsColumnNames = true;
DataSet dataSet = excelReader.AsDataSet();
// Write all sheets that are specified in 'SheetNames' to the data store
foreach (DataTable table in dataSet.Tables)
{
bool keep = StringUtilities.IndexOfCaseInsensitive(this.SheetNames, table.TableName) != -1;
if (keep)
{
TruncateDates(table);
dataStore.WriteTableRaw(table);
}
}
// Close the reader and free resources.
excelReader.Close();
}
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
/// <param name="dataStore">The data store.</param>
/// <exception cref="ApsimXException">
/// Could not find model data table: + ObservedTableName
/// or
/// Could not find observed data table: + ObservedTableName
/// </exception>
public void Run(IStorageReader dataStore)
{
if (PredictedTableName != null && ObservedTableName != null)
{
dataStore.DeleteTable(this.Name);
DataTable predictedDataNames = dataStore.RunQuery("PRAGMA table_info(" + PredictedTableName + ")");
DataTable observedDataNames = dataStore.RunQuery("PRAGMA table_info(" + ObservedTableName + ")");
if (predictedDataNames == null)
{
throw new ApsimXException(this, "Could not find model data table: " + ObservedTableName);
}
if (observedDataNames == null)
{
throw new ApsimXException(this, "Could not find observed data table: " + ObservedTableName);
}
IEnumerable <string> commonCols = from p in predictedDataNames.AsEnumerable()
join o in observedDataNames.AsEnumerable() on p["name"] equals o["name"]
select p["name"] as string;
StringBuilder query = new StringBuilder("SELECT ");
foreach (string s in commonCols)
{
if (s == FieldNameUsedForMatch || s == FieldName2UsedForMatch || s == FieldName3UsedForMatch)
{
query.Append("I.'@field', ");
}
else
{
query.Append("I.'@field' AS 'Observed.@field', R.'@field' AS 'Predicted.@field', ");
}
query.Replace("@field", s);
}
query.Append("FROM " + ObservedTableName + " I INNER JOIN " + PredictedTableName + " R USING (SimulationID) WHERE I.'@match1' = R.'@match1'");
if (FieldName2UsedForMatch != null && FieldName2UsedForMatch != string.Empty)
{
query.Append(" AND I.'@match2' = R.'@match2'");
}
if (FieldName3UsedForMatch != null && FieldName3UsedForMatch != string.Empty)
{
query.Append(" AND I.'@match3' = R.'@match3'");
}
query.Replace(", FROM", " FROM"); // get rid of the last comma
query.Replace("I.'SimulationID' AS 'Observed.SimulationID', R.'SimulationID' AS 'Predicted.SimulationID'", "I.'SimulationID' AS 'SimulationID'");
query = query.Replace("@match1", FieldNameUsedForMatch);
query = query.Replace("@match2", FieldName2UsedForMatch);
query = query.Replace("@match3", FieldName3UsedForMatch);
if (Parent is Folder)
{
// Limit it to particular simulations in scope.
List <string> simulationNames = new List <string>();
foreach (Experiment experiment in Apsim.FindAll(this, typeof(Experiment)))
{
simulationNames.AddRange(experiment.GetSimulationNames());
}
foreach (Simulation simulation in Apsim.FindAll(this, typeof(Simulation)))
{
if (!(simulation.Parent is Experiment))
{
simulationNames.Add(simulation.Name);
}
}
query.Append(" AND I.SimulationID in (");
foreach (string simulationName in simulationNames)
{
if (simulationName != simulationNames[0])
{
query.Append(',');
}
query.Append(dataStore.GetSimulationID(simulationName));
}
query.Append(")");
}
DataTable predictedObservedData = dataStore.RunQuery(query.ToString());
if (predictedObservedData != null)
{
predictedObservedData.TableName = this.Name;
dataStore.WriteTableRaw(predictedObservedData);
}
else
{
// Determine what went wrong.
DataTable predictedData = dataStore.RunQuery("SELECT * FROM " + PredictedTableName);
DataTable observedData = dataStore.RunQuery("SELECT * FROM " + ObservedTableName);
if (predictedData == null || predictedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any predicted data.");
}
else if (observedData == null || observedData.Rows.Count == 0)
{
throw new Exception(Name + ": Cannot find any observed data in node: " + ObservedTableName + ". Check for missing observed file or move " + ObservedTableName + " to top of child list under DataStore (order is important!)");
}
else
{
throw new Exception(Name + ": Observed data was found but didn't match the predicted values. Make sure the values in the SimulationName column match the simulation names in the user interface. Also ensure column names in the observed file match the APSIM report column names.");
}
}
}
}
