public void TestHypersampling() { Dictionary <string, string> parameters = new Dictionary <string, string>(); parameters.Add("precision", "50"); HyperSampling sampling = new HyperSampling(); sampling.setSamplingParameters(parameters); testNumeric(sampling, EXPECTED_HYPERSAMPLING_50); parameters.Clear(); parameters.Add("precision", "40"); sampling.setSamplingParameters(parameters); testNumeric(sampling, EXPECTED_HYPERSAMPLING_40); }
public static bool TestHypersampling(string solver, int expected, int precision) { string loc = resolvePath(solver, "HyperSampling" + precision + ".csv"); Dictionary <string, string> parameters = new Dictionary <string, string>(); parameters.Add("precision", precision.ToString()); HyperSampling sampling = new HyperSampling(); sampling.setSamplingParameters(parameters); List <Configuration> result = testNumeric(sampling); List <Configuration> expectedSample = ConfigurationReader.readConfigurations_Header_CSV(loc, GlobalState.varModel); return(containsAllMeasurements(result, expectedSample) && result.Count == expected); }
/// <summary> /// /// Note: An experimental design might have parameters and also consider only a specific set of numeric options. /// [option1,option3,...,optionN] param1:value param2:value /// </summary> /// <param name="task"></param> /// <returns></returns> public string performOneCommand_ExpDesign(string task) { // splits the task in design and parameters of the design string[] designAndParams = task.Split(new[] { ' ' }, 2); string designName = designAndParams[0];; // parsing of the parameters List <NumericOption> optionsToConsider; Dictionary <string, string> parameter = new Dictionary <string, string>(); List <ConfigurationOption> temp = new List <ConfigurationOption>(); getParametersAndSamplingDomain(task, out parameter, out temp); optionsToConsider = temp.OfType <NumericOption>().ToList(); if (optionsToConsider.Count == 0) { optionsToConsider = GlobalState.varModel.NumericOptions; } ExperimentalDesign expDesign; switch (designName.ToLower()) { case COMMAND_EXPDESIGN_BOXBEHNKEN: expDesign = new BoxBehnkenDesign(); break; case COMMAND_EXPDESIGN_CENTRALCOMPOSITE: expDesign = new CentralCompositeInscribedDesign(); break; case COMMAND_EXPDESIGN_FULLFACTORIAL: expDesign = new FullFactorialDesign(); break; case COMMAND_EXPDESIGN_FACTORIAL: expDesign = new FactorialDesign(); break; case COMMAND_EXPDESIGN_HYPERSAMPLING: expDesign = new HyperSampling(); break; case COMMAND_EXPDESIGN_ONEFACTORATATIME: expDesign = new OneFactorAtATime(); break; case COMMAND_EXPDESIGN_KEXCHANGE: expDesign = new KExchangeAlgorithm(); break; case COMMAND_EXPDESIGN_PLACKETTBURMAN: expDesign = new PlackettBurmanDesign(); break; case COMMAND_EXPDESIGN_RANDOM: expDesign = new RandomSampling(); break; //TODO:hybrids as bin/num //case COMMAND_HYBRID_DISTRIBUTION_AWARE: // addHybridAsNumeric(new DistributionAware(), parameter.ContainsKey("validation"), parameter); // return ""; //case COMMAND_HYBRID_DISTRIBUTION_PRESERVING: // addHybridAsNumeric(new DistributionPreserving(), parameter.ContainsKey("validation"), parameter); // return ""; default: return(task); } if (optionsToConsider.Count > 0) { expDesign.setSamplingDomain(optionsToConsider); } if ((expDesign is KExchangeAlgorithm || expDesign is RandomSampling) && parameter.ContainsKey("sampleSize") && GlobalState.varModel != null) { int maximumNumberNumVariants = computeNumberOfPossibleNumericVariants(GlobalState.varModel); String numberOfSamples; parameter.TryGetValue("sampleSize", out numberOfSamples); if (Double.Parse(numberOfSamples) > maximumNumberNumVariants) { GlobalState.logInfo.logLine("The number of stated numeric variants exceeds the maximum number " + "of possible variants. Only " + maximumNumberNumVariants + " variants are possible. Switching to fullfactorial design."); expDesign = new FullFactorialDesign(); } } expDesign.setSamplingParameters(parameter); if (parameter.ContainsKey("validation")) { this.numericStrategiesValidation.Add(expDesign); } else { this.numericStrategies.Add(expDesign); } return(""); }
/// <summary> /// /// Note: An experimental design might have parameters and also consider only a specific set of numeric options. /// [option1,option3,...,optionN] param1:value param2:value /// </summary> /// <param name="task"></param> /// <returns></returns> private string performOneCommand_ExpDesign(string task) { // splits the task in design and parameters of the design string[] designAndParams = task.Split(new Char[] { ' ' }, 2); string designName = designAndParams[0]; string param = ""; if (designAndParams.Length > 1) param = designAndParams[1]; string[] parameters = param.Split(' '); // parsing of the parameters List<NumericOption> optionsToConsider = new List<NumericOption>(); Dictionary<string, string> parameter = new Dictionary<string, string>(); if (param.Length > 0) { foreach (string par in parameters) { if (par.Contains("[")) { string[] options = par.Substring(1, par.Length - 2).Split(','); foreach (string option in options) { optionsToConsider.Add(GlobalState.varModel.getNumericOption(option)); } } else { if (par.Contains(':')) { string[] nameAndValue = par.Split(':'); parameter.Add(nameAndValue[0], nameAndValue[1]); } else { parameter.Add(par, ""); } } } } if (optionsToConsider.Count == 0) optionsToConsider = GlobalState.varModel.NumericOptions; ExperimentalDesign design = null; switch (designName.ToLower()) { case COMMAND_EXPDESIGN_BOXBEHNKEN: design = new BoxBehnkenDesign(optionsToConsider); break; case COMMAND_EXPDESIGN_CENTRALCOMPOSITE: design = new CentralCompositeInscribedDesign(optionsToConsider); break; case COMMAND_EXPDESIGN_FULLFACTORIAL: design = new FullFactorialDesign(optionsToConsider); break; case "featureInteraction": break; case COMMAND_EXPDESIGN_HYPERSAMPLING: design = new HyperSampling(optionsToConsider); if(parameter.ContainsKey("precision")) ((HyperSampling)design).Precision = Int32.Parse(parameter["precision"]); break; case COMMAND_EXPDESIGN_ONEFACTORATATIME: design = new OneFactorAtATime(optionsToConsider); if(parameter.ContainsKey("distinctValuesPerOption")) ((OneFactorAtATime)design).distinctValuesPerOption = Int32.Parse(parameter["distinctValuesPerOption"]); break; case COMMAND_EXPDESIGN_KEXCHANGE: design = new KExchangeAlgorithm(optionsToConsider); break; case COMMAND_EXPDESIGN_PLACKETTBURMAN: design = new PlackettBurmanDesign(optionsToConsider); if(parameter.ContainsKey("measurements") && parameter.ContainsKey("level")) ((PlackettBurmanDesign)design).setSeed(Int32.Parse(parameter["measurements"]), Int32.Parse(parameter["level"])); break; case COMMAND_EXPDESIGN_RANDOM: design = new RandomSampling(optionsToConsider); break; default: return task; } design.computeDesign(parameter); if (parameter.ContainsKey("validation")) { exp.addNumericSampling_Validation(design.getName()); exp.addNumericalSelection_Validation(design.SelectedConfigurations); } else { exp.addNumericSampling_Learning(design.getName()); exp.addNumericalSelection_Learning(design.SelectedConfigurations); } return ""; }
/// <summary> /// The methods generates based on all binary sampling and 50% hyper sampling configurations for the validation set. /// If we have the true model, we can just compute the true value of the nonfunctional property for a given configuration. /// </summary> private void computeEvaluationDataSetBasedOnTrueModel() { VariantGenerator vg = new VariantGenerator(null); List<List<BinaryOption>> temp = vg.generateAllVariantsFast(GlobalState.varModel); List<List<BinaryOption>> binaryConfigs = new List<List<BinaryOption>>(); //take only 10k if (temp.Count > 1000) { GlobalState.logInfo.log("Found " + temp.Count + " configurations. Use only 1000."); HashSet<int> picked = new HashSet<int>(); Random r = new Random(1); for (int i = 0; i < 1000; i++) { int x = 0; do { x = r.Next(1, temp.Count); } while (picked.Contains(x)); picked.Add(x); binaryConfigs.Add(temp[x]); } temp.Clear(); } else binaryConfigs = temp; exp.addBinarySelection_Validation(binaryConfigs); var expDesign = new HyperSampling(GlobalState.varModel.NumericOptions); expDesign.Precision = 10; expDesign.computeDesign(); exp.addNumericalSelection_Validation(expDesign.SelectedConfigurations); var numericConfigs = expDesign.SelectedConfigurations; foreach (List<BinaryOption> binConfig in binaryConfigs) { if (numericConfigs.Count == 0) { Configuration c = new Configuration(binConfig); c.setMeasuredValue(GlobalState.currentNFP, exp.TrueModel.eval(c)); GlobalState.addConfiguration(c); } foreach (Dictionary<NumericOption, double> numConf in numericConfigs) { Configuration c = new Configuration(binConfig, numConf); c.setMeasuredValue(GlobalState.currentNFP, exp.TrueModel.eval(c)); GlobalState.addConfiguration(c); } } }
public static List<Configuration> buildConfigs(VariabilityModel vm, List<SamplingStrategies> strategies) { List<Configuration> result = new List<Configuration>(); VariantGenerator vg = new VariantGenerator(null); ExperimentalDesign design = null; List<List<BinaryOption>> binaryConfigs = new List<List<BinaryOption>>(); List<Dictionary<NumericOption, Double>> numericConfigs = new List<Dictionary<NumericOption, double>>(); foreach (SamplingStrategies strat in strategies) { switch (strat) { //Binary sampling heuristics case SamplingStrategies.ALLBINARY: binaryConfigs.AddRange(vg.generateAllVariantsFast(vm)); break; case SamplingStrategies.BINARY_RANDOM: binaryConfigs.AddRange(vg.generateRandomVariants(GlobalState.varModel, binaryThreshold, binaryModulu)); break; case SamplingStrategies.OPTIONWISE: FeatureWise fw = new FeatureWise(); binaryConfigs.AddRange(fw.generateFeatureWiseConfigurations(GlobalState.varModel)); break; case SamplingStrategies.PAIRWISE: PairWise pw = new PairWise(); binaryConfigs.AddRange(pw.generatePairWiseVariants(GlobalState.varModel)); break; case SamplingStrategies.NEGATIVE_OPTIONWISE: NegFeatureWise neg = new NegFeatureWise();//2nd option: neg.generateNegativeFWAllCombinations(GlobalState.varModel)); binaryConfigs.AddRange(neg.generateNegativeFW(GlobalState.varModel)); break; //Experimental designs for numeric options case SamplingStrategies.BOXBEHNKEN: if (optionsToConsider.ContainsKey(SamplingStrategies.BOXBEHNKEN)) design = new BoxBehnkenDesign(optionsToConsider[SamplingStrategies.BOXBEHNKEN]); else design = new BoxBehnkenDesign(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.BOXBEHNKEN]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.CENTRALCOMPOSITE: if (optionsToConsider.ContainsKey(SamplingStrategies.CENTRALCOMPOSITE)) design = new CentralCompositeInscribedDesign(optionsToConsider[SamplingStrategies.CENTRALCOMPOSITE]); else design = new CentralCompositeInscribedDesign(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.CENTRALCOMPOSITE]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.FULLFACTORIAL: if (optionsToConsider.ContainsKey(SamplingStrategies.FULLFACTORIAL)) design = new FullFactorialDesign(optionsToConsider[SamplingStrategies.FULLFACTORIAL]); else design = new FullFactorialDesign(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.FULLFACTORIAL]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.HYPERSAMPLING: if (optionsToConsider.ContainsKey(SamplingStrategies.HYPERSAMPLING)) design = new HyperSampling(optionsToConsider[SamplingStrategies.HYPERSAMPLING]); else design = new HyperSampling(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.HYPERSAMPLING]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.ONEFACTORATATIME: if (optionsToConsider.ContainsKey(SamplingStrategies.ONEFACTORATATIME)) design = new OneFactorAtATime(optionsToConsider[SamplingStrategies.ONEFACTORATATIME]); else design = new OneFactorAtATime(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.ONEFACTORATATIME]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.KEXCHANGE: if (optionsToConsider.ContainsKey(SamplingStrategies.KEXCHANGE)) design = new KExchangeAlgorithm(optionsToConsider[SamplingStrategies.KEXCHANGE]); else design = new KExchangeAlgorithm(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.KEXCHANGE]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.PLACKETTBURMAN: if (optionsToConsider.ContainsKey(SamplingStrategies.PLACKETTBURMAN)) design = new PlackettBurmanDesign(optionsToConsider[SamplingStrategies.PLACKETTBURMAN]); else design = new PlackettBurmanDesign(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.PLACKETTBURMAN]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; case SamplingStrategies.RANDOM: if (optionsToConsider.ContainsKey(SamplingStrategies.RANDOM)) design = new RandomSampling(optionsToConsider[SamplingStrategies.RANDOM]); else design = new RandomSampling(vm.NumericOptions); foreach (Dictionary<string, string> expDesignParamSet in parametersOfExpDesigns[SamplingStrategies.RANDOM]) { design.computeDesign(expDesignParamSet); numericConfigs.AddRange(design.SelectedConfigurations); } break; } } foreach (List<BinaryOption> binConfig in binaryConfigs) { if (numericConfigs.Count == 0) { Configuration c = new Configuration(binConfig); result.Add(c); } foreach (Dictionary<NumericOption, double> numConf in numericConfigs) { Configuration c = new Configuration(binConfig, numConf); result.Add(c); } } return result.Distinct().ToList(); }