/// <summary>
/// Create series definitions assuming the vary by fields are text fields in the table.
/// </summary>
/// <param name="reader">The reader to read from.</param>
/// <param name="varyByFieldNames">The vary by fields.</param>
/// <param name="whereClauseForInScopeData">An SQL WHERE clause for rows that are in scope.</param>
private List <SeriesDefinition> CreateDefinitionsFromFieldInTable(IStorageReader reader, List <string> varyByFieldNames, string whereClauseForInScopeData)
{
List <SeriesDefinition> definitions = new List <SeriesDefinition>();
var fieldsThatExist = reader.ColumnNames(TableName);
var varyByThatExistInTable = varyByFieldNames.Where(v => fieldsThatExist.Contains(v)).ToList();
var validValuesForEachVaryByField = new List <List <string> >();
foreach (var varyByFieldName in varyByThatExistInTable)
{
var data = reader.GetData(TableName,
fieldNames: new string[] { varyByFieldName },
filter: whereClauseForInScopeData,
distinct: true);
var values = DataTableUtilities.GetColumnAsStrings(data, varyByFieldName).Distinct().ToList();
validValuesForEachVaryByField.Add(values);
}
foreach (var combination in MathUtilities.AllCombinationsOf(validValuesForEachVaryByField.ToArray(), reverse:true))
{
var descriptors = new List <SimulationDescription.Descriptor>();
for (int i = 0; i < combination.Count; i++)
{
descriptors.Add(new SimulationDescription.Descriptor(varyByThatExistInTable[i],
combination[i]));
}
definitions.Add(new SeriesDefinition(this, whereClauseForInScopeData, Filter, descriptors));
}
return(definitions);
}
/// <summary>
/// If a simulation description has the same descriptor more than once,
/// split it into multiple descriptions.
/// </summary>
/// <remarks>
/// A simulation description can have multiple zones
/// e.g.
/// Sim1 Descriptors: SimulationName=abc, Zone=field1, Zone=field2, x=1, x=2
/// Need to split this into 4 separate simulation descriptions:
/// Sim1 Descriptors: SimulationName=abc, Zone=field1, x=1
/// Sim2 Descriptors: SimulationName=abc, Zone=field1, x=2
/// Sim3 Descriptors: SimulationName=abc, Zone=field2, x=1
/// Sim4 Descriptors: SimulationName=abc, Zone=field2f, x=2
/// </remarks>
/// <param name="simulationDescriptions">Simulation descriptions.</param>
private void SplitDescriptionsWithSameDescriptors(List <SimulationDescription> simulationDescriptions)
{
var newList = new List <SimulationDescription>();
foreach (var simulationDescription in simulationDescriptions)
{
var descriptors = new List <List <SimulationDescription.Descriptor> >();
var descriptorGroups = simulationDescription.Descriptors.GroupBy(d => d.Name);
foreach (var group in descriptorGroups)
{
descriptors.Add(group.ToList());
}
var allCombinations = MathUtilities.AllCombinationsOf(descriptors.ToArray());
foreach (var combination in allCombinations)
{
newList.Add(new SimulationDescription(null, simulationDescription.Name)
{
Descriptors = combination
});
}
}
simulationDescriptions.Clear();
simulationDescriptions.AddRange(newList);
}
/// <summary>
/// Calculate a list of fall combinations of factors.
/// </summary>
private List <List <CompositeFactor> > CalculateAllCombinations()
{
Factors Factors = Apsim.Child(this, typeof(Factors)) as Factors;
// Create a list of list of factorValues so that we can do permutations of them.
List <List <CompositeFactor> > allValues = new List <List <CompositeFactor> >();
if (Factors != null)
{
foreach (Factor factor in Factors.factors)
{
if (factor.Enabled)
{
allValues.Add(factor.GetCompositeFactors());
}
}
var allCombinations = MathUtilities.AllCombinationsOf <CompositeFactor>(allValues.ToArray());
// Remove disabled simulations.
if (DisabledSimNames != null)
{
allCombinations.RemoveAll(comb => DisabledSimNames.Contains(GetName(comb)));
}
return(allCombinations);
}
else
{
return(null);
}
}
/// <summary>
/// Return a list of list of factorvalue objects for all permutations.
/// </summary>
public List <List <FactorValue> > AllCombinations()
{
Factors Factors = Apsim.Child(this, typeof(Factors)) as Factors;
// Create a list of list of factorValues so that we can do permutations of them.
List <List <FactorValue> > allValues = new List <List <FactorValue> >();
if (Factors != null)
{
bool doFullFactorial = false;
foreach (Factor factor in Factors.factors)
{
List <FactorValue> factorValues = factor.CreateValues();
allValues.Add(factorValues);
doFullFactorial = doFullFactorial || factorValues.Count > 1;
}
if (doFullFactorial)
{
return(MathUtilities.AllCombinationsOf <FactorValue>(allValues.ToArray()));
}
else
{
return(allValues);
}
}
return(null);
}
/// <summary>
/// Return a list of list of factorvalue objects for all permutations.
/// </summary>
public List <List <FactorValue> > AllCombinations()
{
Factors Factors = Apsim.Child(this, typeof(Factors)) as Factors;
// Create a list of list of factorValues so that we can do permutations of them.
List <List <FactorValue> > allValues = new List <List <FactorValue> >();
if (Factors != null)
{
bool doFullFactorial = true;
foreach (Factor factor in Factors.factors)
{
if (factor.Enabled)
{
List <FactorValue> factorValues = factor.CreateValues();
// Iff any of the factors modify the same model (e.g. have a duplicate path), then we do not want to do a full factorial.
// This code should check if there are any such duplicates by checking each path in each factor value in the list of factor
// values for the current factor against each path in each list of factor values in the list of all factors which we have
// already added to the global list of list of factor values.
foreach (FactorValue currentFactorValue in factorValues)
{
foreach (string currentFactorPath in currentFactorValue.Paths)
{
foreach (List <FactorValue> allFactorValues in allValues)
{
foreach (FactorValue globalValue in allFactorValues)
{
foreach (string globalPath in globalValue.Paths)
{
if (string.Equals(globalPath, currentFactorPath, StringComparison.CurrentCulture))
{
doFullFactorial = false;
}
}
}
}
}
}
allValues.Add(factorValues);
}
}
if (doFullFactorial)
{
return(MathUtilities.AllCombinationsOf <FactorValue>(allValues.ToArray()));
}
else
{
return(allValues);
}
}
return(null);
}
/// <summary>Return a enumerable collection of groups where a group is
/// defined by the current index.</summary>
public IEnumerable <IndexedDataTableGroupEnumerator> Groups()
{
SetIndex(null);
List <List <object> > allValues = new List <List <object> >();
foreach (string indexColumnName in indexColumnNames)
{
allValues.Add(Get <object>(indexColumnName).Distinct().ToList());
}
List <List <object> > allPermutations = MathUtilities.AllCombinationsOf <object>(allValues.ToArray());
foreach (var permutation in allPermutations)
{
SetIndex(permutation.ToArray());
if (view.Count > 0)
{
yield return(new IndexedDataTableGroupEnumerator(this, permutation.ToArray()));
}
}
}
/// <summary>
/// Get a list of row filters that define the blocks of data that we have
/// to calculate stats for.
/// </summary>
/// <returns></returns>
private List <string> GetRowFilters(DataTable data)
{
var rowFilters = new List <string>();
List <List <string> > fieldValues = new List <List <string> >();
foreach (var fieldName in FieldNamesToSplitOn)
{
if (!data.Columns.Contains(fieldName))
{
throw new Exception($"Cannot find field {fieldName} in table {data.TableName}");
}
fieldValues.Add(DataTableUtilities.GetColumnAsStrings(data, fieldName).Distinct().ToList());
}
var permutations = MathUtilities.AllCombinationsOf <string>(fieldValues.ToArray());
foreach (var permutation in permutations)
{
rowFilters.Add(CreateRowFilter(permutation, data));
}
return(rowFilters);
}
/// <summary>
/// Get a list of all permutations of child factors and compositefactors.
/// </summary>
internal List <List <CompositeFactor> > GetPermutations()
{
var factors = new List <List <CompositeFactor> >();
foreach (Factor factor in Apsim.Children(this, typeof(Factor)))
{
if (factor.Enabled)
{
factors.Add(factor.GetCompositeFactors());
}
}
var compositeFactors = Apsim.Children(this, typeof(CompositeFactor)).Where(cf => cf.Enabled);
var permutations = new List <List <CompositeFactor> >();
if (compositeFactors.Count() > 0)
{
// Loop through each composite factor and permute with the factor children.
foreach (CompositeFactor compositeFactor in compositeFactors)
{
var valuesToPermutate = new List <List <CompositeFactor> >(factors);
valuesToPermutate.Add(new List <CompositeFactor>()
{
compositeFactor
});
permutations.AddRange(MathUtilities.AllCombinationsOf <CompositeFactor>(valuesToPermutate.ToArray()));
}
}
else
{
permutations = MathUtilities.AllCombinationsOf <CompositeFactor>(factors.ToArray());
}
return(permutations);
}
/// <summary>Get a .db filter for the specified combination.</summary>
/// <param name="experiment">The experiment</param>
/// <param name="combination">The combination</param>
/// <returns>The filter</returns>
private string GetFilter(Experiment experiment, List <FactorAndIndex> combination)
{
// Need to determine all the simulation names that match the specified combination.
// If the combination is just a single factor value then the simulation names will be
// a permutation of that factor value and all other factor values in other factors
// For example:
// IF combination is CVGibe
// THEN filter = SimulationName = 'VanDeldenCvGibePP1' or SimulationName = 'VanDeldenCvGibePP2' or SimulationName = 'VanDeldenCvGibePP3' ...
// If the combintation of 2 factor values for 2 different factors then the filter will
// be a permutation of those 2 factor values and all OTHER factor values in OTHER factors.
// For example:
// IF combintation is CVGibePP1
// THEN filter = SimulationName = 'SimulationName = 'VanDeldenCvGibePP1'
// (This is because the example has no other factors other than CV and PP.
Factors factors = Apsim.Child(experiment as IModel, typeof(Factors)) as Factors;
List <List <KeyValuePair <string, string> > > simulationBits = new List <List <KeyValuePair <string, string> > >();
foreach (Factor factor in factors.factors)
{
List <KeyValuePair <string, string> > names = new List <KeyValuePair <string, string> >();
FactorAndIndex factorAndIndex = combination.Find(f => factor.Name == f.factorName);
if (factorAndIndex == null)
{
if (factor.Children.Count > 0)
{
foreach (IModel child in factor.Children)
{
names.Add(new KeyValuePair <string, string>(factor.Name, child.Name));
}
}
else
{
foreach (FactorValue factorValue in factor.CreateValues())
{
if (factorValue.Values.Count >= 1)
{
names.Add(new KeyValuePair <string, string>(factor.Name, factorValue.Values[0].ToString()));
}
}
}
}
else
{
names.Add(new KeyValuePair <string, string>(factor.Name, factorAndIndex.factorValue));
}
simulationBits.Add(names);
}
List <List <KeyValuePair <string, string> > > combinations = MathUtilities.AllCombinationsOf <KeyValuePair <string, string> >(simulationBits.ToArray());
string filter = string.Empty;
foreach (List <KeyValuePair <string, string> > c in combinations)
{
if (filter != string.Empty)
{
filter += " or ";
}
filter += "SimulationName = '" + experiment.Name;
foreach (KeyValuePair <string, string> pair in c)
{
filter += pair.Key + pair.Value;
}
filter += "'";
}
return(filter);
}
/// <summary>
/// Graph the specified experiment.
/// </summary>
/// <param name="parentExperiment"></param>
/// <param name="ourDefinitions"></param>
private void GraphExperiment(Experiment parentExperiment, List <SeriesDefinition> ourDefinitions)
{
Factors factors = Apsim.Child(parentExperiment as IModel, typeof(Factors)) as Factors;
if (factors != null)
{
// Given this example (from Teff.apsimx).
// Factors
// CV - Gibe, Ziquala, Ayana, 04T19 (4 factor values)
// PP - 1, 2, 3, 4, 5, 6 (6 factor values)
// -----------------------------------------------------------
// If FactorIndexToVaryColours = 0 (index of CV factor)
// FactorIndexToVaryLines = 1 (index of PP factor)
// FactorIndexToVaryMarkers = -1 (doesn't point to a factor)
// Then permutations will be:
// CVGibe & PP1
// CVZiquala & PP2
// CVAyana & PP3
// ... (24 in total - 4 x 6)
// -----------------------------------------------------------
// If FactorIndexToVaryColours = 0 (index of CV factor)
// FactorIndexToVaryLines = -1 (doesn't point to a factor)
// FactorIndexToVaryMarkers = -1 (doesn't point to a factor)
// Then permutations will be:
// CVGibe
// CVZiquala
// CVAyana
// CV04T19 (4 in total - 4)
// -----------------------------------------------------------
// The FactorIndexToVary... variables denote which factors should be
// separate series.
List <List <FactorAndIndex> > factorIndexes = new List <List <FactorAndIndex> >();
for (int f = 0; f != factors.Children.Count; f++)
{
if (FactorIndexToVaryColours == f)
{
CreateFactorAndIndex(factors.Children[FactorIndexToVaryColours] as Factor, factorIndexes,
FactorAndIndex.TypeToVary.Colour);
}
if (FactorIndexToVaryLines == f)
{
CreateFactorAndIndex(factors.Children[FactorIndexToVaryLines] as Factor, factorIndexes,
FactorAndIndex.TypeToVary.Line);
}
if (FactorIndexToVaryMarkers == f)
{
CreateFactorAndIndex(factors.Children[FactorIndexToVaryMarkers] as Factor, factorIndexes,
FactorAndIndex.TypeToVary.Marker);
}
}
List <List <FactorAndIndex> > permutations = MathUtilities.AllCombinationsOf(factorIndexes.ToArray());
// If no 'vary by' were specified then create a dummy one. All data will be on one series.
if (permutations == null || permutations.Count == 0)
{
permutations = new List <List <FactorAndIndex> >();
permutations.Add(new List <FactorAndIndex>());
}
// Loop through all permutations and create a graph series definition for each.
foreach (List <FactorAndIndex> combination in permutations)
{
// Determine the marker, line and colour for this combination.
MarkerType marker = Marker;
LineType line = Line;
int colourIndex = Array.IndexOf(ColourUtilities.Colours, Colour);
if (colourIndex == -1)
{
colourIndex = 0;
}
string seriesName = string.Empty;
for (int i = 0; i < combination.Count; i++)
{
if (combination[i].typeToVary == FactorAndIndex.TypeToVary.Colour)
{
colourIndex = combination[i].factorValueIndex;
}
if (combination[i].typeToVary == FactorAndIndex.TypeToVary.Marker)
{
marker = GetEnumValue <MarkerType>(combination[i].factorValueIndex);
}
if (combination[i].typeToVary == FactorAndIndex.TypeToVary.Line)
{
line = GetEnumValue <LineType>(combination[i].factorValueIndex);
}
seriesName += combination[i].factorName + combination[i].factorValue;
}
string filter = GetFilter(parentExperiment, combination);
CreateDefinitions(parentExperiment.BaseSimulation, seriesName, filter, ref colourIndex,
ref marker, line, ourDefinitions, parentExperiment.Names());
}
}
}
/// <summary>
/// Return all possible factor values for this factor.
/// </summary>
public List <FactorValue> CreateValues()
{
List <FactorValue> factorValues = new List <FactorValue>();
// Example specifications:
// simple specification:
// [SowingRule].Script.SowingDate = 2003-11-01
// [SowingRule].Script.SowingDate = 2003-11-01, 2003-12-20
// range specification:
// [FertiliserRule].Script.ApplicationAmount = 0 to 100 step 20
// model replacement specification:
// [FertiliserRule]
// compound specification has multiple other specifications that
// result in a single factor value being returned.
List <List <PathValuesPair> > allValues = new List <List <PathValuesPair> >();
List <PathValuesPair> fixedValues = new List <PathValuesPair>();
foreach (string specification in Specifications)
{
if (specification.Contains(" to ") &&
specification.Contains(" step "))
{
allValues.Add(ParseRangeSpecification(specification));
}
else
{
List <PathValuesPair> localValues;
if (specification.Contains('='))
{
localValues = ParseSimpleSpecification(specification);
}
else
{
localValues = ParseModelReplacementSpecification(specification);
}
if (localValues.Count == 1)
{
fixedValues.Add(localValues[0]);
}
else if (localValues.Count > 1)
{
allValues.Add(localValues);
}
}
}
// Look for child Factor models.
foreach (Factor childFactor in Apsim.Children(this, typeof(Factor)))
{
foreach (FactorValue childFactorValue in childFactor.CreateValues())
{
childFactorValue.Name = Name + childFactorValue.Name;
factorValues.Add(childFactorValue);
}
}
if (allValues.Count == 0)
{
PathValuesPairToFactorValue(factorValues, fixedValues, null);
}
List <List <PathValuesPair> > allCombinations = MathUtilities.AllCombinationsOf <PathValuesPair>(allValues.ToArray());
if (allCombinations != null)
{
foreach (List <PathValuesPair> combination in allCombinations)
{
PathValuesPairToFactorValue(factorValues, fixedValues, combination);
}
}
return(factorValues);
}
