public void TestSetScalarThenSetVector()
        {
            IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });

            indexedTable.SetIndex(new object[] { 2000, "Name1" });
            indexedTable.Set("A", 1234);                              // scalar
            indexedTable.SetValues("B", new double[] { 1, 2, 3, 4 }); // vector

            Assert.AreEqual(Utilities.TableToString(indexedTable.ToTable()),
                            "Year, Name,   A,    B\r\n" +
                            "2000,Name1,1234,1.000\r\n" +
                            "2000,Name1,1234,2.000\r\n" +
                            "2000,Name1,1234,3.000\r\n" +
                            "2000,Name1,1234,4.000\r\n");
        }
Exemple #2
0
        /// <summary>
        /// The main run method called to fill tables in the specified DataStore.
        /// </summary>
        /// <param name="dataStore">The DataStore to work with</param>
        public void Run(IStorageReader dataStore)
        {
            dataStore.DeleteDataInTable(this.Name);

            DataTable simulationData = dataStore.GetData(TableName, fieldNames: dataStore.GetTableColumns(TableName));

            if (simulationData != null)
            {
                IndexedDataTable simData         = new IndexedDataTable(simulationData, new string[] { FieldToSplitOn });
                IndexedDataTable probabilityData = new IndexedDataTable(new string[] { FieldToSplitOn });

                foreach (var group in simData.Groups())
                {
                    object keyValue = group.IndexValues[0];

                    // Add in our key column
                    probabilityData.SetIndex(new object[] { keyValue });
                    probabilityData.Set <object>(FieldToSplitOn, keyValue);

                    // Add in all other numeric columns.
                    bool haveWrittenProbabilityColumn = false;

                    foreach (DataColumn column in simulationData.Columns)
                    {
                        if (column.DataType == typeof(double))
                        {
                            var values = group.Get <double>(column.ColumnName).ToList();
                            values.Sort();

                            if (!haveWrittenProbabilityColumn)
                            {
                                // Add in the probability column
                                double[] probabilityValues = MathUtilities.ProbabilityDistribution(values.Count, this.Exceedence);
                                probabilityData.SetValues("Probability", probabilityValues);
                                haveWrittenProbabilityColumn = true;
                            }

                            probabilityData.SetValues(column.ColumnName, values);
                        }
                    }
                }

                // Write the stats data to the DataStore
                DataTable t = probabilityData.ToTable();
                t.TableName = this.Name;
                dataStore.WriteTable(t);
            }
        }
        public void TestNoIndexSetScalarThenSetVector()
        {
            IndexedDataTable indexedTable = new IndexedDataTable(null);

            indexedTable.Set("A", 1234);                              // scalar
            indexedTable.SetValues("B", new double[] { 1, 2, 3, 4 }); // vector

            Assert.IsTrue(
                Utilities.CreateTable(new string[]                      { "A", "B" },
                                      new List <object[]> {
                new object[] { 1234, 1 },
                new object[] { 1234, 2 },
                new object[] { 1234, 3 },
                new object[] { 1234, 4 }
            })
                .IsSame(indexedTable.ToTable()));
        }
        public void TestSetVectorThenSetScalar()
        {
            IndexedDataTable indexedTable = new IndexedDataTable(new string[] { "Year", "Name" });

            indexedTable.SetIndex(new object[] { 2000, "Name1" });
            indexedTable.SetValues("A", new double[] { 1, 2, 3, 4 }); // vector
            indexedTable.Set("B", 1234);                              // scalar

            Assert.IsTrue(
                Utilities.CreateTable(new string[]                      { "Year", "Name", "A", "B" },
                                      new List <object[]> {
                new object[] { 2000, "Name1", 1, 1234 },
                new object[] { 2000, "Name1", 2, 1234 },
                new object[] { 2000, "Name1", 3, 1234 },
                new object[] { 2000, "Name1", 4, 1234 }
            })
                .IsSame(indexedTable.ToTable()));
        }
Exemple #5
0
        /// <summary>Main run method for performing our post simulation calculations</summary>
        public void Run()
        {
            // If the predicted table has not been modified, don't do anything.
            // This can happen if other simulations were run but the Morris model was not.
            if (dataStore?.Writer != null && !dataStore.Writer.TablesModified.Contains(TableName))
            {
                return;
            }

            DataTable predictedData = dataStore.Reader.GetData(TableName);

            if (predictedData != null)
            {
                // Determine how many aggregation values we have per simulation
                DataView view = new DataView(predictedData);
                view.RowFilter    = "SimulationName='" + Name + "Simulation1'";
                AggregationValues = DataTableUtilities.GetColumnAsStrings(view, AggregationVariableName);

                // Create a table of all predicted values
                DataTable predictedValues = new DataTable();

                List <string> descriptiveColumnNames = new List <string>();
                List <string> variableNames          = new List <string>();
                foreach (string aggregationValue in AggregationValues)
                {
                    string value = aggregationValue;
                    if (DateTime.TryParse(value, out DateTime date))
                    {
                        value = date.ToString("yyyy-MM-dd");
                    }
                    view.RowFilter = $"{AggregationVariableName}='{value}'";

                    foreach (DataColumn predictedColumn in view.Table.Columns)
                    {
                        if (predictedColumn.DataType == typeof(double))
                        {
                            double[] values = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
                            if (values.Distinct().Count() == 1)
                            {
                                if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
                                {
                                    descriptiveColumnNames.Add(predictedColumn.ColumnName);
                                }
                            }
                            else
                            {
                                DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + "_" + value, values);
                                if (!variableNames.Contains(predictedColumn.ColumnName))
                                {
                                    variableNames.Add(predictedColumn.ColumnName);
                                }
                            }
                        }
                    }
                }

                // Run R
                DataTable eeDataRaw;
                DataTable statsDataRaw;
                RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);

                // Get ee data from R and store in ee table.
                // EE data from R looks like:
                // "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
                // - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
                // - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
                // - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
                // - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
                DataView         eeView     = new DataView(eeDataRaw);
                IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", AggregationVariableName });

                // Create a path variable.
                var pathValues = Enumerable.Range(1, NumPaths).ToArray();

                foreach (var parameter in Parameters)
                {
                    foreach (DataColumn column in predictedValues.Columns)
                    {
                        eeView.RowFilter = "variable = '" + column.ColumnName + "'";
                        if (eeView.Count != NumPaths)
                        {
                            throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
                        }
                        string aggregationValue = StringUtilities.GetAfter(column.ColumnName, "_");
                        string variableName     = StringUtilities.RemoveAfter(column.ColumnName, '_');

                        eeTableKey.SetIndex(new object[] { parameter.Name, aggregationValue });

                        List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
                        for (int i = 0; i < values.Count; i++)
                        {
                            values[i] = Math.Abs(values[i]);
                        }
                        var runningMean = MathUtilities.RunningAverage(values);

                        eeTableKey.SetValues("Path", pathValues);
                        eeTableKey.SetValues(variableName + ".MuStar", runningMean);
                    }
                }
                DataTable eeTable = eeTableKey.ToTable();
                eeTable.TableName = Name + "PathAnalysis";

                // Get stats data from R and store in MuStar table.
                // Stats data coming back from R looks like:
                // "mu", "mustar", "sigma", "param","variable"
                // -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
                // -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
                // -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
                // 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
                // -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
                // 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
                // 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
                // -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
                IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
                    "Parameter", AggregationVariableName
                });

                foreach (DataRow row in statsDataRaw.Rows)
                {
                    string variable         = row["variable"].ToString();
                    string aggregationValue = StringUtilities.GetAfter(variable, "_");
                    variable = StringUtilities.RemoveAfter(variable, '_');
                    tableKey.SetIndex(new object[] { row["param"], aggregationValue });

                    tableKey.Set(variable + ".Mu", row["mu"]);
                    tableKey.Set(variable + ".MuStar", row["mustar"]);
                    tableKey.Set(variable + ".Sigma", row["sigma"]);

                    // Need to bring in the descriptive values.
                    view.RowFilter = $"{AggregationVariableName}='{aggregationValue}'";
                    foreach (var descriptiveColumnName in descriptiveColumnNames)
                    {
                        var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
                        if (values.Distinct().Count() == 1)
                        {
                            tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
                        }
                    }
                }
                DataTable muStarTable = tableKey.ToTable();
                muStarTable.TableName = Name + "Statistics";

                dataStore.Writer.WriteTable(eeTable);
                dataStore.Writer.WriteTable(muStarTable);
            }
        }
Exemple #6
0
        /// <summary>Main run method for performing our post simulation calculations</summary>
        /// <param name="dataStore">The data store.</param>
        public void Run(IDataStore dataStore)
        {
            DataTable predictedData = dataStore.Reader.GetData("Report", filter: "SimulationName LIKE '" + Name + "%'", orderBy: "SimulationID");

            if (predictedData != null)
            {
                // Determine how many years we have per simulation
                DataView view = new DataView(predictedData);
                view.RowFilter = "SimulationName='" + Name + "Simulation1'";
                Years          = DataTableUtilities.GetColumnAsIntegers(view, "Clock.Today.Year");

                // Create a table of all predicted values
                DataTable predictedValues = new DataTable();

                List <string> descriptiveColumnNames = new List <string>();
                List <string> variableNames          = new List <string>();
                foreach (double year in Years)
                {
                    view.RowFilter = "Clock.Today.Year=" + year;

                    foreach (DataColumn predictedColumn in view.Table.Columns)
                    {
                        if (predictedColumn.DataType == typeof(double))
                        {
                            double[] valuesForYear = DataTableUtilities.GetColumnAsDoubles(view, predictedColumn.ColumnName);
                            if (valuesForYear.Distinct().Count() == 1)
                            {
                                if (!descriptiveColumnNames.Contains(predictedColumn.ColumnName))
                                {
                                    descriptiveColumnNames.Add(predictedColumn.ColumnName);
                                }
                            }
                            else
                            {
                                DataTableUtilities.AddColumn(predictedValues, predictedColumn.ColumnName + year, valuesForYear);
                                if (!variableNames.Contains(predictedColumn.ColumnName))
                                {
                                    variableNames.Add(predictedColumn.ColumnName);
                                }
                            }
                        }
                    }
                }

                // Run R
                DataTable eeDataRaw;
                DataTable statsDataRaw;
                RunRPostSimulation(predictedValues, out eeDataRaw, out statsDataRaw);

                // Get ee data from R and store in ee table.
                // EE data from R looks like:
                // "ResidueWt", "FASW", "CN2", "Cona", "variable","path"
                // - 22.971008269563,0.00950570342209862,-0.00379987333757356,56.7587080430652,"FallowEvaporation1996",1
                // - 25.790599484188, 0.0170777988614538, -0.0265991133629069,58.0240658644712,"FallowEvaporation1996",2
                // - 26.113599477728, 0.0113851992409871, 0.0113996200126667,57.9689677010766,"FallowEvaporation1996",3
                // - 33.284199334316, 0.0323193916349732, -0.334388853704853,60.5376820772641,"FallowEvaporation1996",4
                DataView         eeView     = new DataView(eeDataRaw);
                IndexedDataTable eeTableKey = new IndexedDataTable(new string[] { "Parameter", "Year" });

                // Create a path variable.
                var pathValues = Enumerable.Range(1, NumPaths).ToArray();

                foreach (var parameter in Parameters)
                {
                    foreach (DataColumn column in predictedValues.Columns)
                    {
                        eeView.RowFilter = "variable = '" + column.ColumnName + "'";
                        if (eeView.Count != NumPaths)
                        {
                            throw new Exception("Found only " + eeView.Count + " paths for variable " + column.ColumnName + " in ee table");
                        }
                        int    year         = Convert.ToInt32(column.ColumnName.Substring(column.ColumnName.Length - 4));
                        string variableName = column.ColumnName.Substring(0, column.ColumnName.Length - 4);

                        eeTableKey.SetIndex(new object[] { parameter.Name, year });

                        List <double> values = DataTableUtilities.GetColumnAsDoubles(eeView, parameter.Name).ToList();
                        for (int i = 0; i < values.Count; i++)
                        {
                            values[i] = Math.Abs(values[i]);
                        }
                        var runningMean = MathUtilities.RunningAverage(values);

                        eeTableKey.SetValues("Path", pathValues);
                        eeTableKey.SetValues(variableName + ".MuStar", runningMean);
                    }
                }
                DataTable eeTable = eeTableKey.ToTable();
                eeTable.TableName = Name + "PathAnalysis";

                // Get stats data from R and store in MuStar table.
                // Stats data coming back from R looks like:
                // "mu", "mustar", "sigma", "param","variable"
                // -30.7331368183818, 30.7331368183818, 5.42917964248002,"ResidueWt","FallowEvaporation1996"
                // -0.0731299918470997,0.105740687296631,0.450848277601353, "FASW","FallowEvaporation1996"
                // -0.83061431285624,0.839772007599748, 1.75541097254145, "CN2","FallowEvaporation1996"
                // 62.6942591520838, 62.6942591520838, 5.22778043503867, "Cona","FallowEvaporation1996"
                // -17.286285468283, 19.4018404625051, 24.1361388348929,"ResidueWt","FallowRunoff1996"
                // 8.09850688306722, 8.09852589447407, 15.1988107373113, "FASW","FallowRunoff1996"
                // 18.6196168461051, 18.6196168461051, 15.1496277765849, "CN2","FallowRunoff1996"
                // -7.12794888887507, 7.12794888887507, 5.54014788597839, "Cona","FallowRunoff1996"
                IndexedDataTable tableKey = new IndexedDataTable(new string[2] {
                    "Parameter", "Year"
                });

                foreach (DataRow row in statsDataRaw.Rows)
                {
                    string variable = row["variable"].ToString();
                    int    year     = Convert.ToInt32(variable.Substring(variable.Length - 4));
                    variable = variable.Substring(0, variable.Length - 4);
                    tableKey.SetIndex(new object[] { row["param"], year });

                    tableKey.Set(variable + ".Mu", row["mu"]);
                    tableKey.Set(variable + ".MuStar", row["mustar"]);
                    tableKey.Set(variable + ".Sigma", row["sigma"]);

                    // Need to bring in the descriptive values.
                    view.RowFilter = "Clock.Today.Year=" + year;
                    foreach (var descriptiveColumnName in descriptiveColumnNames)
                    {
                        var values = DataTableUtilities.GetColumnAsStrings(view, descriptiveColumnName);
                        if (values.Distinct().Count() == 1)
                        {
                            tableKey.Set(descriptiveColumnName, view[0][descriptiveColumnName]);
                        }
                    }
                }
                DataTable muStarTable = tableKey.ToTable();
                muStarTable.TableName = Name + "Statistics";

                dataStore.Writer.WriteTable(eeTable);
                dataStore.Writer.WriteTable(muStarTable);
            }
        }