public void TestIterateThroughGroups()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
indexedTable.Set("B", 1234); // scalar
indexedTable.SetIndex(new object[] { 2001, "Name2" });
indexedTable.SetValues("A", new double[] { 5, 6, 7, 8 }); // vector
indexedTable.Set("B", 5678); // scalar
int i = 1;
foreach (var group in indexedTable.Groups())
{
var a = group.Get <double>("A");
if (i == 1)
{
Assert.AreEqual(a, new double[] { 1, 2, 3, 4 });
}
else
{
Assert.AreEqual(a, new double[] { 5, 6, 7, 8 });
}
i++;
}
}
public void TestChangeIndex()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
indexedTable.Set("B", 1234); // scalar
indexedTable.SetIndex(new object[] { 2001, "Name2" });
indexedTable.SetValues("A", new double[] { 5, 6, 7, 8 }); // vector
indexedTable.Set("B", 5678); // scalar
Assert.IsTrue(
Utilities.CreateTable(new string[] { "Year", "Name", "A", "B" },
new List <object[]> {
new object[] { 2000, "Name1", 1, 1234 },
new object[] { 2000, "Name1", 2, 1234 },
new object[] { 2000, "Name1", 3, 1234 },
new object[] { 2000, "Name1", 4, 1234 },
new object[] { 2001, "Name2", 5, 5678 },
new object[] { 2001, "Name2", 6, 5678 },
new object[] { 2001, "Name2", 7, 5678 },
new object[] { 2001, "Name2", 8, 5678 }
})
.IsSame(indexedTable.ToTable()));
}
public void TestGetColumn()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
indexedTable.Set("B", 1234); // scalar
indexedTable.SetIndex(new object[] { 2001, "Name2" });
indexedTable.SetValues("A", new double[] { 5, 6, 7, 8 }); // vector
indexedTable.Set("B", 5678); // scalar
IndexedDataTable indexedTable2 = new IndexedDataTable(indexedTable.ToTable(), new string[] { "Year" });
indexedTable2.SetIndex(new object[] { 2000 });
var a = indexedTable2.Get <double>("A");
Assert.AreEqual(a, new double[] { 1, 2, 3, 4 });
var names = indexedTable2.Get <string>("Name");
Assert.AreEqual(names, new string[] { "Name1", "Name1", "Name1", "Name1" });
indexedTable2.SetIndex(new object[] { 2001 });
var b = indexedTable2.Get <int>("B");
Assert.AreEqual(b, new int[] { 5678, 5678, 5678, 5678 });
}
public void TestChangeIndex()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
indexedTable.Set("B", 1234); // scalar
indexedTable.SetIndex(new object[] { 2001, "Name2" });
indexedTable.SetValues("A", new double[] { 5, 6, 7, 8 }); // vector
indexedTable.Set("B", 5678); // scalar
string expected = ReflectionUtilities.GetResourceAsString("UnitTests.APSIMShared.TestChangeIndex.Expected.txt");
Assert.AreEqual(Utilities.TableToString(indexedTable.ToTable()), expected);
}
/// <summary>Main run method for performing our calculations and storing data.</summary>
public void Run()
{
// If the target table has not been modified during the simulation run, don't do anything.
if (dataStore?.Writer != null && !dataStore.Writer.TablesModified.Contains(TableName))
{
return;
}
if (string.IsNullOrWhiteSpace(TableName))
{
throw new Exception(string.Format("Error in probability model {0}: TableName is null", Name));
}
else if (!dataStore.Reader.TableNames.Contains(TableName))
{
throw new Exception(string.Format("Error in probability model {0}: table '{1}' does not exist in the database.", Name, TableName));
}
DataTable simulationData = dataStore.Reader.GetData(TableName, fieldNames: dataStore.Reader.ColumnNames(TableName));
if (simulationData != null)
{
IndexedDataTable simData = new IndexedDataTable(simulationData, new string[] { FieldToSplitOn });
IndexedDataTable probabilityData = new IndexedDataTable(new string[] { FieldToSplitOn });
foreach (var group in simData.Groups())
{
object keyValue = group.IndexValues[0];
// Add in our key column
probabilityData.SetIndex(new object[] { keyValue });
probabilityData.Set <object>(FieldToSplitOn, keyValue);
// Add in all other numeric columns.
bool haveWrittenProbabilityColumn = false;
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
var values = group.Get <double>(column.ColumnName).ToList();
values.Sort();
if (!haveWrittenProbabilityColumn)
{
// Add in the probability column
double[] probabilityValues = MathUtilities.ProbabilityDistribution(values.Count, this.Exceedence);
probabilityData.SetValues("Probability", probabilityValues);
haveWrittenProbabilityColumn = true;
}
probabilityData.SetValues(column.ColumnName, values);
}
}
}
// Write the stats data to the DataStore
DataTable t = probabilityData.ToTable();
t.TableName = this.Name;
dataStore.Writer.WriteTable(t);
}
}
public void TestChangeIndex()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
indexedTable.Set("B", 1234); // scalar
indexedTable.SetIndex(new object[] { 2001, "Name2" });
indexedTable.SetValues("A", new double[] { 5, 6, 7, 8 }); // vector
indexedTable.Set("B", 5678); // scalar
Assert.AreEqual(Utilities.TableToString(indexedTable.ToTable()),
"Year, Name, A, B\r\n" +
"2000,Name1,1.000,1234\r\n" +
"2000,Name1,2.000,1234\r\n" +
"2000,Name1,3.000,1234\r\n" +
"2000,Name1,4.000,1234\r\n" +
"2001,Name2,5.000,5678\r\n" +
"2001,Name2,6.000,5678\r\n" +
"2001,Name2,7.000,5678\r\n" +
"2001,Name2,8.000,5678\r\n");
}
public void TestSetScalarThenSetVector()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.Set("A", 1234); // scalar
indexedTable.SetValues("B", new double[] { 1, 2, 3, 4 }); // vector
Assert.AreEqual(Utilities.TableToString(indexedTable.ToTable()),
"Year, Name, A, B\r\n" +
"2000,Name1,1234,1.000\r\n" +
"2000,Name1,1234,2.000\r\n" +
"2000,Name1,1234,3.000\r\n" +
"2000,Name1,1234,4.000\r\n");
}
/// <summary>
/// The main run method called to fill tables in the specified DataStore.
/// </summary>
/// <param name="dataStore">The DataStore to work with</param>
public void Run(IStorageReader dataStore)
{
dataStore.DeleteDataInTable(this.Name);
DataTable simulationData = dataStore.GetData(TableName, fieldNames: dataStore.GetTableColumns(TableName));
if (simulationData != null)
{
IndexedDataTable simData = new IndexedDataTable(simulationData, new string[] { FieldToSplitOn });
IndexedDataTable probabilityData = new IndexedDataTable(new string[] { FieldToSplitOn });
foreach (var group in simData.Groups())
{
object keyValue = group.IndexValues[0];
// Add in our key column
probabilityData.SetIndex(new object[] { keyValue });
probabilityData.Set <object>(FieldToSplitOn, keyValue);
// Add in all other numeric columns.
bool haveWrittenProbabilityColumn = false;
foreach (DataColumn column in simulationData.Columns)
{
if (column.DataType == typeof(double))
{
var values = group.Get <double>(column.ColumnName).ToList();
values.Sort();
if (!haveWrittenProbabilityColumn)
{
// Add in the probability column
double[] probabilityValues = MathUtilities.ProbabilityDistribution(values.Count, this.Exceedence);
probabilityData.SetValues("Probability", probabilityValues);
haveWrittenProbabilityColumn = true;
}
probabilityData.SetValues(column.ColumnName, values);
}
}
}
// Write the stats data to the DataStore
DataTable t = probabilityData.ToTable();
t.TableName = this.Name;
dataStore.WriteTable(t);
}
}
public void TestSetScalarThenSetVector()
{
IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });
indexedTable.SetIndex(new object[] { 2000, "Name1" });
indexedTable.Set("A", 1234); // scalar
indexedTable.SetValues("B", new double[] { 1, 2, 3, 4 }); // vector
Assert.IsTrue(
Utilities.CreateTable(new string[] { "Year", "Name", "A", "B" },
new List <object[]> {
new object[] { 2000, "Name1", 1234, 1 },
new object[] { 2000, "Name1", 1234, 2 },
new object[] { 2000, "Name1", 1234, 3 },
new object[] { 2000, "Name1", 1234, 4 }
})
.IsSame(indexedTable.ToTable()));
}
/// <summary>Main run method for performing our post simulation calculations</summary>
public void Run()
{
// If the predicted table has not been modified, don't do anything.
// This can happen if other simulations were run but the Morris model was not.
if (dataStore?.Writer != null && !dataStore.Writer.TablesModified.Contains(TableName))
{
return;
}
DataTable predictedData = dataStore.Reader.GetData(TableName);
if (predictedData != null)
{
// Determine how many aggregation values we have per simulation
DataView view = new DataView(predictedData);
view.RowFilter = "SimulationName='" + Name + "Simulation1'";
AggregationValues = DataTableUtilities.GetColumnAsStrings(view, AggregationVariableName);
// Create a table of all predicted values
DataTable predictedValues = new DataTable();
List <string> descriptiveColumnNames = new List <string>();
List <string> variableNames = new List <string>();
foreach (string aggregationValue in AggregationValues)
{
string value = aggregationValue;
if (DateTime.TryParse(value, out DateTime date))
{
value = date.ToString("yyyy-MM-dd");
}
view.RowFilter = $"{AggregationVariableName}='{value}'";
foreach (DataColumn predictedColumn in view.Table.Columns)
{
if (predictedColumn.DataType == typeof(double))
{
double[] values = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
if (values.Distinct().Count() == 1)
{
if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
{
descriptiveColumnNames.Add(predictedColumn.ColumnName);
}
}
else
{
DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + "_" + value, values);
if (!variableNames.Contains(predictedColumn.ColumnName))
{
variableNames.Add(predictedColumn.ColumnName);
}
}
}
}
}
// Run R
DataTable eeDataRaw;
DataTable statsDataRaw;
RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);
// Get ee data from R and store in ee table.
// EE data from R looks like:
// "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
// - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
// - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
// - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
// - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
DataView eeView = new DataView(eeDataRaw);
IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", AggregationVariableName });
// Create a path variable.
var pathValues = Enumerable.Range(1, NumPaths).ToArray();
foreach (var parameter in Parameters)
{
foreach (DataColumn column in predictedValues.Columns)
{
eeView.RowFilter = "variable = '" + column.ColumnName + "'";
if (eeView.Count != NumPaths)
{
throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
}
string aggregationValue = StringUtilities.GetAfter(column.ColumnName, "_");
string variableName = StringUtilities.RemoveAfter(column.ColumnName, '_');
eeTableKey.SetIndex(new object[] { parameter.Name, aggregationValue });
List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
for (int i = 0; i < values.Count; i++)
{
values[i] = Math.Abs(values[i]);
}
var runningMean = MathUtilities.RunningAverage(values);
eeTableKey.SetValues("Path", pathValues);
eeTableKey.SetValues(variableName + ".MuStar", runningMean);
}
}
DataTable eeTable = eeTableKey.ToTable();
eeTable.TableName = Name + "PathAnalysis";
// Get stats data from R and store in MuStar table.
// Stats data coming back from R looks like:
// "mu", "mustar", "sigma", "param","variable"
// -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
// -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
// -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
// 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
// -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
// 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
// 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
// -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
"Parameter", AggregationVariableName
});
foreach (DataRow row in statsDataRaw.Rows)
{
string variable = row["variable"].ToString();
string aggregationValue = StringUtilities.GetAfter(variable, "_");
variable = StringUtilities.RemoveAfter(variable, '_');
tableKey.SetIndex(new object[] { row["param"], aggregationValue });
tableKey.Set(variable + ".Mu", row["mu"]);
tableKey.Set(variable + ".MuStar", row["mustar"]);
tableKey.Set(variable + ".Sigma", row["sigma"]);
// Need to bring in the descriptive values.
view.RowFilter = $"{AggregationVariableName}='{aggregationValue}'";
foreach (var descriptiveColumnName in descriptiveColumnNames)
{
var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
if (values.Distinct().Count() == 1)
{
tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
}
}
}
DataTable muStarTable = tableKey.ToTable();
muStarTable.TableName = Name + "Statistics";
dataStore.Writer.WriteTable(eeTable);
dataStore.Writer.WriteTable(muStarTable);
}
}
/// <summary>Main run method for performing our post simulation calculations</summary>
/// <param name="dataStore">The data store.</param>
public void Run(IDataStore dataStore)
{
DataTable predictedData = dataStore.Reader.GetData("Report", filter: "SimulationName LIKE '" + Name + "%'", orderBy: "SimulationID");
if (predictedData != null)
{
// Determine how many years we have per simulation
DataView view = new DataView(predictedData);
view.RowFilter = "SimulationName='" + Name + "Simulation1'";
Years = DataTableUtilities.GetColumnAsIntegers(view, "Clock.Today.Year");
// Create a table of all predicted values
DataTable predictedValues = new DataTable();
List <string> descriptiveColumnNames = new List <string>();
List <string> variableNames = new List <string>();
foreach (double year in Years)
{
view.RowFilter = "Clock.Today.Year=" + year;
foreach (DataColumn predictedColumn in view.Table.Columns)
{
if (predictedColumn.DataType == typeof(double))
{
double[] valuesForYear = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
if (valuesForYear.Distinct().Count() == 1)
{
if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
{
descriptiveColumnNames.Add(predictedColumn.ColumnName);
}
}
else
{
DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + year, valuesForYear);
if (!variableNames.Contains(predictedColumn.ColumnName))
{
variableNames.Add(predictedColumn.ColumnName);
}
}
}
}
}
// Run R
DataTable eeDataRaw;
DataTable statsDataRaw;
RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);
// Get ee data from R and store in ee table.
// EE data from R looks like:
// "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
// - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
// - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
// - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
// - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
DataView eeView = new DataView(eeDataRaw);
IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", "Year" });
// Create a path variable.
var pathValues = Enumerable.Range(1, NumPaths).ToArray();
foreach (var parameter in Parameters)
{
foreach (DataColumn column in predictedValues.Columns)
{
eeView.RowFilter = "variable = '" + column.ColumnName + "'";
if (eeView.Count != NumPaths)
{
throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
}
int year = Convert.ToInt32(column.ColumnName.Substring(column.ColumnName.Length - 4));
string variableName = column.ColumnName.Substring(0, column.ColumnName.Length - 4);
eeTableKey.SetIndex(new object[] { parameter.Name, year });
List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
for (int i = 0; i < values.Count; i++)
{
values[i] = Math.Abs(values[i]);
}
var runningMean = MathUtilities.RunningAverage(values);
eeTableKey.SetValues("Path", pathValues);
eeTableKey.SetValues(variableName + ".MuStar", runningMean);
}
}
DataTable eeTable = eeTableKey.ToTable();
eeTable.TableName = Name + "PathAnalysis";
// Get stats data from R and store in MuStar table.
// Stats data coming back from R looks like:
// "mu", "mustar", "sigma", "param","variable"
// -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
// -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
// -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
// 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
// -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
// 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
// 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
// -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
"Parameter", "Year"
});
foreach (DataRow row in statsDataRaw.Rows)
{
string variable = row["variable"].ToString();
int year = Convert.ToInt32(variable.Substring(variable.Length - 4));
variable = variable.Substring(0, variable.Length - 4);
tableKey.SetIndex(new object[] { row["param"], year });
tableKey.Set(variable + ".Mu", row["mu"]);
tableKey.Set(variable + ".MuStar", row["mustar"]);
tableKey.Set(variable + ".Sigma", row["sigma"]);
// Need to bring in the descriptive values.
view.RowFilter = "Clock.Today.Year=" + year;
foreach (var descriptiveColumnName in descriptiveColumnNames)
{
var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
if (values.Distinct().Count() == 1)
{
tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
}
}
}
DataTable muStarTable = tableKey.ToTable();
muStarTable.TableName = Name + "Statistics";
dataStore.Writer.WriteTable(eeTable);
dataStore.Writer.WriteTable(muStarTable);
}
}
