/// <summary>Initializes a new instance of the
/// <see cref="TestableRandomSampling"/>
/// class.</summary>
/// <param name="randomSampling">
/// The random sampling to test.</param>
/// <param name="populationSize">The expected population size.</param>
/// <param name="sampleSize">The expected sample size.</param>
/// <param name="inclusionProbabilities">
/// The expected inclusion probabilities.
/// </param>
/// <param name="goodnessOfFitCriticalValue">
/// A quantile of the chi-squared distribution with a number of
/// degrees of freedom equal to <see cref="PopulationSize"/>
/// minus <c>1</c>. To serve as the critical value for the Pearson's
/// chi-squared test whose null hypothesis assume that the
/// the <paramref name="inclusionProbabilities"/> hold true
/// when samples are drawn via the <paramref name="randomSampling"/>
/// instance.
/// </param>
/// <seealso href="https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test"/>
public TestableRandomSampling(
RandomSampling randomSampling,
int populationSize,
int sampleSize,
DoubleMatrix inclusionProbabilities,
double goodnessOfFitCriticalValue)
{
this.randomSampling = randomSampling;
this.popilationSize = populationSize;
this.sampleSize = sampleSize;
this.inclusionProbabilities = inclusionProbabilities;
this.goodnessOfFitCriticalValue = goodnessOfFitCriticalValue;
}
public static bool TestNumericRandom(string solver, int expected, int seed, int sampleSize)
{
string loc = resolvePath(solver, "RandomSampling" + seed.ToString() + sampleSize.ToString() + ".csv");
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("seed", seed.ToString());
parameters.Add("sampleSize", sampleSize.ToString());
RandomSampling sampling = new RandomSampling();
sampling.setSamplingParameters(parameters);
List <Configuration> result = testNumeric(sampling);
List <Configuration> expectedSample = ConfigurationReader.readConfigurations_Header_CSV(loc, GlobalState.varModel);
return(containsAllMeasurements(result, expectedSample) && result.Count == expected);
}
public void TestNumericRandom()
{
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("seed", "1");
parameters.Add("sampleSize", "12");
RandomSampling sampling = new RandomSampling();
sampling.setSamplingParameters(parameters);
testNumeric(sampling, EXPECTED_RANDOM_12_1);
parameters.Clear();
parameters.Add("seed", "0");
parameters.Add("sampleSize", "10");
sampling.setSamplingParameters(parameters);
testNumeric(sampling, EXPECTED_RANDOM_10_0);
}
/// <summary>
///
/// Note: An experimental design might have parameters and also consider only a specific set of numeric options.
/// [option1,option3,...,optionN] param1:value param2:value
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public string performOneCommand_ExpDesign(string task)
{
// splits the task in design and parameters of the design
string[] designAndParams = task.Split(new[] { ' ' }, 2);
string designName = designAndParams[0];;
// parsing of the parameters
List <NumericOption> optionsToConsider;
Dictionary <string, string> parameter = new Dictionary <string, string>();
List <ConfigurationOption> temp = new List <ConfigurationOption>();
getParametersAndSamplingDomain(task, out parameter, out temp);
optionsToConsider = temp.OfType <NumericOption>().ToList();
if (optionsToConsider.Count == 0)
{
optionsToConsider = GlobalState.varModel.NumericOptions;
}
ExperimentalDesign expDesign;
switch (designName.ToLower())
{
case COMMAND_EXPDESIGN_BOXBEHNKEN:
expDesign = new BoxBehnkenDesign();
break;
case COMMAND_EXPDESIGN_CENTRALCOMPOSITE:
expDesign = new CentralCompositeInscribedDesign();
break;
case COMMAND_EXPDESIGN_FULLFACTORIAL:
expDesign = new FullFactorialDesign();
break;
case COMMAND_EXPDESIGN_FACTORIAL:
expDesign = new FactorialDesign();
break;
case COMMAND_EXPDESIGN_HYPERSAMPLING:
expDesign = new HyperSampling();
break;
case COMMAND_EXPDESIGN_ONEFACTORATATIME:
expDesign = new OneFactorAtATime();
break;
case COMMAND_EXPDESIGN_KEXCHANGE:
expDesign = new KExchangeAlgorithm();
break;
case COMMAND_EXPDESIGN_PLACKETTBURMAN:
expDesign = new PlackettBurmanDesign();
break;
case COMMAND_EXPDESIGN_RANDOM:
expDesign = new RandomSampling();
break;
//TODO:hybrids as bin/num
//case COMMAND_HYBRID_DISTRIBUTION_AWARE:
// addHybridAsNumeric(new DistributionAware(), parameter.ContainsKey("validation"), parameter);
// return "";
//case COMMAND_HYBRID_DISTRIBUTION_PRESERVING:
// addHybridAsNumeric(new DistributionPreserving(), parameter.ContainsKey("validation"), parameter);
// return "";
default:
return(task);
}
if (optionsToConsider.Count > 0)
{
expDesign.setSamplingDomain(optionsToConsider);
}
if ((expDesign is KExchangeAlgorithm || expDesign is RandomSampling) &&
parameter.ContainsKey("sampleSize") && GlobalState.varModel != null)
{
int maximumNumberNumVariants = computeNumberOfPossibleNumericVariants(GlobalState.varModel);
String numberOfSamples;
parameter.TryGetValue("sampleSize", out numberOfSamples);
if (Double.Parse(numberOfSamples) > maximumNumberNumVariants)
{
GlobalState.logInfo.logLine("The number of stated numeric variants exceeds the maximum number "
+ "of possible variants. Only " + maximumNumberNumVariants
+ " variants are possible. Switching to fullfactorial design.");
expDesign = new FullFactorialDesign();
}
}
expDesign.setSamplingParameters(parameter);
if (parameter.ContainsKey("validation"))
{
this.numericStrategiesValidation.Add(expDesign);
}
else
{
this.numericStrategies.Add(expDesign);
}
return("");
}
/// <summary>
///
/// Note: An experimental design might have parameters and also consider only a specific set of numeric options.
/// [option1,option3,...,optionN] param1:value param2:value
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
private string performOneCommand_ExpDesign(string task)
{
// splits the task in design and parameters of the design
string[] designAndParams = task.Split(new Char[] { ' ' }, 2);
string designName = designAndParams[0];
string param = "";
if (designAndParams.Length > 1)
param = designAndParams[1];
string[] parameters = param.Split(' ');
// parsing of the parameters
List<NumericOption> optionsToConsider = new List<NumericOption>();
Dictionary<string, string> parameter = new Dictionary<string, string>();
if (param.Length > 0)
{
foreach (string par in parameters)
{
if (par.Contains("["))
{
string[] options = par.Substring(1, par.Length - 2).Split(',');
foreach (string option in options)
{
optionsToConsider.Add(GlobalState.varModel.getNumericOption(option));
}
}
else
{
if (par.Contains(':'))
{
string[] nameAndValue = par.Split(':');
parameter.Add(nameAndValue[0], nameAndValue[1]);
}
else
{
parameter.Add(par, "");
}
}
}
}
if (optionsToConsider.Count == 0)
optionsToConsider = GlobalState.varModel.NumericOptions;
ExperimentalDesign design = null;
switch (designName.ToLower())
{
case COMMAND_EXPDESIGN_BOXBEHNKEN:
design = new BoxBehnkenDesign(optionsToConsider);
break;
case COMMAND_EXPDESIGN_CENTRALCOMPOSITE:
design = new CentralCompositeInscribedDesign(optionsToConsider);
break;
case COMMAND_EXPDESIGN_FULLFACTORIAL:
design = new FullFactorialDesign(optionsToConsider);
break;
case "featureInteraction":
break;
case COMMAND_EXPDESIGN_HYPERSAMPLING:
design = new HyperSampling(optionsToConsider);
if(parameter.ContainsKey("precision"))
((HyperSampling)design).Precision = Int32.Parse(parameter["precision"]);
break;
case COMMAND_EXPDESIGN_ONEFACTORATATIME:
design = new OneFactorAtATime(optionsToConsider);
if(parameter.ContainsKey("distinctValuesPerOption"))
((OneFactorAtATime)design).distinctValuesPerOption = Int32.Parse(parameter["distinctValuesPerOption"]);
break;
case COMMAND_EXPDESIGN_KEXCHANGE:
design = new KExchangeAlgorithm(optionsToConsider);
break;
case COMMAND_EXPDESIGN_PLACKETTBURMAN:
design = new PlackettBurmanDesign(optionsToConsider);
if(parameter.ContainsKey("measurements") && parameter.ContainsKey("level"))
((PlackettBurmanDesign)design).setSeed(Int32.Parse(parameter["measurements"]), Int32.Parse(parameter["level"]));
break;
case COMMAND_EXPDESIGN_RANDOM:
design = new RandomSampling(optionsToConsider);
break;
default:
return task;
}
design.computeDesign(parameter);
if (parameter.ContainsKey("validation"))
{
exp.addNumericSampling_Validation(design.getName());
exp.addNumericalSelection_Validation(design.SelectedConfigurations);
}
else
{
exp.addNumericSampling_Learning(design.getName());
exp.addNumericalSelection_Learning(design.SelectedConfigurations);
}
return "";
}
public static List<Configuration> buildConfigs(VariabilityModel vm, List<SamplingStrategies> strategies)
{
List<Configuration> result = new List<Configuration>();
VariantGenerator vg = new VariantGenerator(null);
ExperimentalDesign design = null;
List<List<BinaryOption>> binaryConfigs = new List<List<BinaryOption>>();
List<Dictionary<NumericOption, Double>> numericConfigs = new List<Dictionary<NumericOption, double>>();
foreach (SamplingStrategies strat in strategies)
{
switch (strat)
{
//Binary sampling heuristics
case SamplingStrategies.ALLBINARY:
binaryConfigs.AddRange(vg.generateAllVariantsFast(vm));
break;
case SamplingStrategies.BINARY_RANDOM:
binaryConfigs.AddRange(vg.generateRandomVariants(GlobalState.varModel, binaryThreshold, binaryModulu));
break;
case SamplingStrategies.OPTIONWISE:
FeatureWise fw = new FeatureWise();
binaryConfigs.AddRange(fw.generateFeatureWiseConfigurations(GlobalState.varModel));
break;
case SamplingStrategies.PAIRWISE:
PairWise pw = new PairWise();
binaryConfigs.AddRange(pw.generatePairWiseVariants(GlobalState.varModel));
break;
case SamplingStrategies.NEGATIVE_OPTIONWISE:
NegFeatureWise neg = new NegFeatureWise();//2nd option: neg.generateNegativeFWAllCombinations(GlobalState.varModel));
binaryConfigs.AddRange(neg.generateNegativeFW(GlobalState.varModel));
break;
//Experimental designs for numeric options
case SamplingStrategies.BOXBEHNKEN:
if (optionsToConsider.ContainsKey(SamplingStrategies.BOXBEHNKEN))
design = new BoxBehnkenDesign(optionsToConsider[SamplingStrategies.BOXBEHNKEN]);
else
design = new BoxBehnkenDesign(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.BOXBEHNKEN])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.CENTRALCOMPOSITE:
if (optionsToConsider.ContainsKey(SamplingStrategies.CENTRALCOMPOSITE))
design = new CentralCompositeInscribedDesign(optionsToConsider[SamplingStrategies.CENTRALCOMPOSITE]);
else
design = new CentralCompositeInscribedDesign(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.CENTRALCOMPOSITE])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.FULLFACTORIAL:
if (optionsToConsider.ContainsKey(SamplingStrategies.FULLFACTORIAL))
design = new FullFactorialDesign(optionsToConsider[SamplingStrategies.FULLFACTORIAL]);
else
design = new FullFactorialDesign(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.FULLFACTORIAL])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.HYPERSAMPLING:
if (optionsToConsider.ContainsKey(SamplingStrategies.HYPERSAMPLING))
design = new HyperSampling(optionsToConsider[SamplingStrategies.HYPERSAMPLING]);
else
design = new HyperSampling(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.HYPERSAMPLING])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.ONEFACTORATATIME:
if (optionsToConsider.ContainsKey(SamplingStrategies.ONEFACTORATATIME))
design = new OneFactorAtATime(optionsToConsider[SamplingStrategies.ONEFACTORATATIME]);
else
design = new OneFactorAtATime(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.ONEFACTORATATIME])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.KEXCHANGE:
if (optionsToConsider.ContainsKey(SamplingStrategies.KEXCHANGE))
design = new KExchangeAlgorithm(optionsToConsider[SamplingStrategies.KEXCHANGE]);
else
design = new KExchangeAlgorithm(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.KEXCHANGE])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.PLACKETTBURMAN:
if (optionsToConsider.ContainsKey(SamplingStrategies.PLACKETTBURMAN))
design = new PlackettBurmanDesign(optionsToConsider[SamplingStrategies.PLACKETTBURMAN]);
else
design = new PlackettBurmanDesign(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.PLACKETTBURMAN])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
case SamplingStrategies.RANDOM:
if (optionsToConsider.ContainsKey(SamplingStrategies.RANDOM))
design = new RandomSampling(optionsToConsider[SamplingStrategies.RANDOM]);
else
design = new RandomSampling(vm.NumericOptions);
foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.RANDOM])
{
design.computeDesign(expDesignParamSet);
numericConfigs.AddRange(design.SelectedConfigurations);
}
break;
}
}
foreach (List<BinaryOption> binConfig in binaryConfigs)
{
if (numericConfigs.Count == 0)
{
Configuration c = new Configuration(binConfig);
result.Add(c);
}
foreach (Dictionary<NumericOption, double> numConf in numericConfigs)
{
Configuration c = new Configuration(binConfig, numConf);
result.Add(c);
}
}
return result.Distinct().ToList();
}
