/// <summary> /// Given a predictor type returns a set of all permissible learners (with their sweeper params, if defined). /// </summary> /// <returns>Array of viable learners.</returns> public static SuggestedRecipe.SuggestedLearner[] AllowedLearners(IHostEnvironment env, MacroUtils.TrainerKinds trainerKind) { //not all learners advertised in the API are available in CORE. var catalog = ModuleCatalog.CreateInstance(env); var availableLearnersList = catalog.AllEntryPoints().Where( x => x.InputKinds?.FirstOrDefault(i => i == typeof(CommonInputs.ITrainerInput)) != null); var learners = new List <SuggestedRecipe.SuggestedLearner>(); var type = typeof(CommonInputs.ITrainerInput); var trainerTypes = typeof(Experiment).Assembly.GetTypes() .Where(p => type.IsAssignableFrom(p) && MacroUtils.IsTrainerOfKind(p, trainerKind)); foreach (var tt in trainerTypes) { var sweepParams = AutoMlUtils.GetSweepRanges(tt); var epInputObj = (CommonInputs.ITrainerInput)tt.GetConstructor(Type.EmptyTypes)?.Invoke(new object[] { }); var sl = new SuggestedRecipe.SuggestedLearner { PipelineNode = new TrainerPipelineNode(epInputObj, sweepParams), LearnerName = tt.Name }; if (sl.PipelineNode != null && availableLearnersList.FirstOrDefault(l => l.Name.Equals(sl.PipelineNode.GetEpName())) != null) { learners.Add(sl); } } return(learners.ToArray()); }
private void SampleHyperparameters(RecipeInference.SuggestedRecipe.SuggestedLearner learner, PipelinePattern[] history) { // If first time optimizing hyperparams, create new hyperparameter sweeper. if (!_hyperSweepers.ContainsKey(learner.LearnerName)) { var sps = AutoMlUtils.ConvertToComponentFactories(learner.PipelineNode.SweepParams); if (sps.Length > 0) { _hyperSweepers[learner.LearnerName] = new KdoSweeper(Env, new KdoSweeper.Arguments { SweptParameters = sps, NumberInitialPopulation = Math.Max(_remainingThirdStageTrials, 2) }); } else { _hyperSweepers[learner.LearnerName] = new FalseSweeper(); } } var sweeper = _hyperSweepers[learner.LearnerName]; var historyToUse = history.Where(p => p.Learner.LearnerName == learner.LearnerName).ToArray(); if (_currentStage == (int)Stages.Third) { _remainingThirdStageTrials--; historyToUse = new PipelinePattern[0]; if (_remainingThirdStageTrials < 1) { _currentStage++; } } SampleHyperparameters(learner, sweeper, IsMaximizingMetric, historyToUse); }
private void RandomlyPerturbSweepableHyperparameters(IEnumerable <TlcModule.SweepableParamAttribute> sweepParams) { foreach (var param in sweepParams) { switch (param) { case TlcModule.SweepableDiscreteParamAttribute disParam: Env.Assert(disParam.Options.Length > 0, $"Trying to sweep over discrete parameter, {disParam.Name}, with no options."); disParam.RawValue = Host.Rand.Next(disParam.Options.Length); break; case TlcModule.SweepableFloatParamAttribute floParam: var fvg = AutoMlUtils.ToIValueGenerator(floParam); floParam.RawValue = ((IParameterValue <float>)fvg.CreateFromNormalized(Host.Rand.NextSingle())).Value; break; case TlcModule.SweepableLongParamAttribute lonParam: var lvg = AutoMlUtils.ToIValueGenerator(lonParam); lonParam.RawValue = ((IParameterValue <long>)lvg.CreateFromNormalized(Host.Rand.NextSingle())).Value; break; default: throw new NotSupportedException($"Unknown type of sweepable parameter attribute: {param.GetType()}"); } } }
public virtual void UpdateLearners(RecipeInference.SuggestedRecipe.SuggestedLearner[] availableLearners) { AvailableLearners = availableLearners; foreach (var learner in AvailableLearners) { AutoMlUtils.PopulateSweepableParams(learner); } }
private TransformInference.SuggestedTransform[] SampleTransforms(out long transformsBitMask) { // For now, return all transforms. var sampledTransforms = AvailableTransforms.ToList(); transformsBitMask = AutoMlUtils.TransformsToBitmask(sampledTransforms.ToArray()); // Add final features concat transform. sampledTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData, DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms)); return(sampledTransforms.ToArray()); }
/// <summary> /// Runs a train-test experiment on the current pipeline, through entrypoints. /// </summary> public void RunTrainTestExperiment(IDataView trainData, IDataView testData, SupportedMetric metric, MacroUtils.TrainerKinds trainerKind, out double testMetricValue, out double trainMetricValue) { var experiment = CreateTrainTestExperiment(trainData, testData, trainerKind, true, out var trainTestOutput); experiment.Run(); var dataOut = experiment.GetOutput(trainTestOutput.OverallMetrics); var dataOutTraining = experiment.GetOutput(trainTestOutput.TrainingOverallMetrics); testMetricValue = AutoMlUtils.ExtractValueFromIdv(_env, dataOut, metric.Name); trainMetricValue = AutoMlUtils.ExtractValueFromIdv(_env, dataOutTraining, metric.Name); }
public TransformPipelineNode(CommonInputs.ITransformInput entryPointObj, IEnumerable <TlcModule.SweepableParamAttribute> sweepParams = null, CommonInputs.ITrainerInput subTrainerObj = null) { var newEp = CloneEntryPoint(entryPointObj); _entryPointObj = newEp ?? entryPointObj; if (subTrainerObj != null) { _subTrainerObj = CloneEntryPoint(subTrainerObj); } SweepParams = sweepParams?.Select(p => p.Clone()).ToArray() ?? AutoMlUtils.GetSweepRanges(_entryPointObj.GetType()); }
public TrainerPipelineNode(CommonInputs.ITrainerInput entryPointObj, IEnumerable <TlcModule.SweepableParamAttribute> sweepParams = null, ParameterSet hyperParameterSet = null) { var newEp = CloneEntryPoint(entryPointObj); _entryPointObj = newEp ?? entryPointObj; SweepParams = sweepParams?.Select(p => p.Clone()).ToArray() ?? AutoMlUtils.GetSweepRanges(_entryPointObj.GetType()); HyperSweeperParamSet = hyperParameterSet?.Clone(); // Make sure sweep params and param set are consistent. if (HyperSweeperParamSet != null) { PropagateParamSetValues(HyperSweeperParamSet, SweepParams); UpdateProperties(); } }
protected double[] LearnerHistoryToWeights(PipelinePattern[] history, bool isMaximizingMetric) { int numLearners = AvailableLearners.Length; double[] weights = new double[numLearners]; int[] counts = new int[numLearners]; Dictionary <string, int> labelToIndex = new Dictionary <string, int>(); double maxWeight = history.Length > 0 ? history.Max(w => w.PerformanceSummary.MetricValue) : 0d; // Map categorical values to their index for (int j = 0; j < numLearners; j++) { labelToIndex[AvailableLearners[j].LearnerName] = j; } // Add mass according to performance foreach (var pipeline in history) { if (AvailableLearners.All(l => l.LearnerName != pipeline.Learner.LearnerName)) { continue; } weights[labelToIndex[pipeline.Learner.LearnerName]] += AutoMlUtils.ProcessWeight(pipeline.PerformanceSummary.MetricValue, maxWeight, isMaximizingMetric); counts[labelToIndex[pipeline.Learner.LearnerName]]++; } // Take average mass for each learner for (int i = 0; i < weights.Length; i++) { weights[i] /= counts[i] > 0 ? counts[i] : 1; } // If any learner has not been seen, default its average to 1.0 // to encourage exploration of untried algorithms. for (int i = 0; i < weights.Length; i++) { weights[i] += counts[i] == 0 ? 1 : 0; } // Normalize weights to sum to one and return return(SweeperProbabilityUtils.Normalize(weights)); }
public virtual void SetSpace(TransformInference.SuggestedTransform[] availableTransforms, RecipeInference.SuggestedRecipe.SuggestedLearner[] availableLearners, Func <PipelinePattern, long, bool> pipelineVerifier, IDataView originalData, IDataView fullyTransformedData, AutoInference.DependencyMap dependencyMapping, bool isMaximizingMetric) { AvailableLearners = availableLearners; AvailableTransforms = availableTransforms; PipelineVerifier = pipelineVerifier; OriginalData = originalData; FullyTransformedData = fullyTransformedData; DependencyMapping = dependencyMapping; IsMaximizingMetric = isMaximizingMetric; foreach (var learner in AvailableLearners) { AutoMlUtils.PopulateSweepableParams(learner); } }
public override PipelinePattern[] GetNextCandidates(IEnumerable <PipelinePattern> history, int numCandidates, RoleMappedData dataRoles) { var candidates = new List <PipelinePattern>(); DataRoles = dataRoles; while (candidates.Count < numCandidates) { Contracts.Assert(0 <= _currentLearnerIndex && _currentLearnerIndex < AvailableLearners.Length); // Select hyperparameters and transforms based on learner and history. PipelinePattern pipeline; int count = 0; bool valid; var learner = AvailableLearners[_currentLearnerIndex]; // Make sure sweep paramater values exist; if not, populate them from learner object. if (learner.PipelineNode.SweepParams.Any(p => p.RawValue == null)) { AutoMlUtils.PopulateSweepableParams(learner); } do { // Make sure transforms set is valid. Repeat until passes verifier. pipeline = new PipelinePattern(SampleTransforms(out var transformsBitMask), learner, "", Env); valid = PipelineVerifier(pipeline, transformsBitMask); count++; } while (!valid && count <= 1000); // Keep only valid pipelines. if (valid) { candidates.Add(pipeline); } // Update current index _currentLearnerIndex = (_currentLearnerIndex + 1) % AvailableLearners.Length; } return(candidates.ToArray()); }
protected void SampleHyperparameters(RecipeInference.SuggestedRecipe.SuggestedLearner learner, ISweeper sweeper, bool isMaximizingMetric, PipelinePattern[] history) { // Make sure there are hyperparameters to sweep over. var hyperParams = learner.PipelineNode.SweepParams; if (hyperParams.Length == 0) { return; } // Get new set of hyperparameter values. var proposedParamSet = sweeper.ProposeSweeps(1, AutoMlUtils.ConvertToRunResults(history, isMaximizingMetric)).First(); Env.Assert(proposedParamSet != null && proposedParamSet.All(ps => hyperParams.Any(hp => hp.Name == ps.Name))); // Associate proposed param set with learner, so that smart hyperparam // sweepers (like KDO) can map them back. learner.PipelineNode.HyperSweeperParamSet = proposedParamSet; var generatorSet = hyperParams.Select(AutoMlUtils.ToIValueGenerator).ToArray(); var values = SweeperProbabilityUtils.ParameterSetAsFloatArray(Host, generatorSet, proposedParamSet, false); // Update hyperparameters. for (int i = 0; i < hyperParams.Length; i++) { if (hyperParams[i] is TlcModule.SweepableDiscreteParamAttribute dp) { hyperParams[i].RawValue = (int)values[i]; } else { hyperParams[i].RawValue = values[i]; } } }
private PipelinePattern[] NextCandidates(PipelinePattern[] history, int numCandidates, bool defaultHyperParams = false, bool uniformRandomTransforms = false) { const int maxNumberAttempts = 10; double[] learnerWeights = LearnerHistoryToWeights(history, IsMaximizingMetric); var candidates = new List <PipelinePattern>(); var sampledLearners = new RecipeInference.SuggestedRecipe.SuggestedLearner[numCandidates]; if (_currentStage == (int)Stages.Second || _currentStage == (int)Stages.Third) { // Select remaining learners in round-robin fashion. for (int i = 0; i < numCandidates; i++) { sampledLearners[i] = AvailableLearners[i % AvailableLearners.Length].Clone(); } } else { // Select learners, based on weights. var indices = ProbUtils.SampleCategoricalDistribution(numCandidates, learnerWeights); foreach (var item in indices.Select((idx, i) => new { idx, i })) { sampledLearners[item.i] = AvailableLearners[item.idx].Clone(); } } // Select hyperparameters and transforms based on learner and history. foreach (var learner in sampledLearners) { PipelinePattern pipeline; int count = 0; bool valid; string hashKey; if (!defaultHyperParams) { SampleHyperparameters(learner, history); } else { AutoMlUtils.PopulateSweepableParams(learner); } do { // Make sure transforms set is valid and have not seen pipeline before. // Repeat until passes or runs out of chances. pipeline = new PipelinePattern( SampleTransforms(learner, history, out var transformsBitMask, uniformRandomTransforms), learner, "", Env); hashKey = GetHashKey(transformsBitMask, learner); valid = PipelineVerifier(pipeline, transformsBitMask) && !VisitedPipelines.Contains(hashKey); count++; } while (!valid && count <= maxNumberAttempts); // If maxed out chances and at second stage, move onto next stage. if (count >= maxNumberAttempts && _currentStage == (int)Stages.Second) { _currentStage++; } // Keep only valid pipelines. if (valid) { VisitedPipelines.Add(hashKey); candidates.Add(pipeline); } } return(candidates.ToArray()); }
private TransformInference.SuggestedTransform[] SampleTransforms(RecipeInference.SuggestedRecipe.SuggestedLearner learner, PipelinePattern[] history, out long transformsBitMask, bool uniformRandomSampling = false) { var sampledTransforms = new List <TransformInference.SuggestedTransform>( AutoMlUtils.GetMandatoryTransforms(AvailableTransforms)); var remainingAvailableTransforms = AvailableTransforms.Where(t => !sampledTransforms.Any(t.Equals)).ToArray(); var mask = AutoMlUtils.TransformsToBitmask(sampledTransforms.ToArray()); foreach (var transform in remainingAvailableTransforms) { // Add pseudo-mass to encourage sampling of untried transforms. double maxWeight = history.Length > 0 ? history.Max(w => w.PerformanceSummary.MetricValue) : 0d; double allWeight = Math.Max(maxWeight, 1d); double learnerWeight = Math.Max(maxWeight, 1d); int allCounts = 1; int learnerCounts = 1; // Add mass according to performance. foreach (var pipeline in history) { if (pipeline.Transforms.Any(transform.Equals)) { allWeight += AutoMlUtils.ProcessWeight(pipeline.PerformanceSummary.MetricValue, maxWeight, IsMaximizingMetric); allCounts++; if (pipeline.Learner.LearnerName == learner.LearnerName) { learnerWeight += pipeline.PerformanceSummary.MetricValue; learnerCounts++; } } } // Take average mass as weight, and take convex combination of // learner-specific weight and unconditioned weight. allWeight /= allCounts > 0 ? allCounts : 1; learnerWeight /= learnerCounts > 0 ? learnerCounts : 1; var lambda = MathUtils.Sigmoid(learnerCounts - 3); var combinedWeight = uniformRandomSampling ? 0.5 : lambda * learnerWeight + (1 - lambda) * allWeight; // Sample transform according to combined weight. if (ProbUtils.SampleUniform() <= combinedWeight / maxWeight) { mask |= 1L << transform.AtomicGroupId; } } // Add all chosen transforms. sampledTransforms.AddRange(remainingAvailableTransforms.Where(t => AutoMlUtils.AtomicGroupPresent(mask, t.AtomicGroupId))); // Add final features concat transform. NOTE: computed bitmask should always // exclude the final features concat. If we forget to exclude that one, will // cause an error in verification, since it isn't included in the original // dependency mapping (i.e., its level isn't in the dictionary). sampledTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData, DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms, DataRoles)); transformsBitMask = mask; return(sampledTransforms.ToArray()); }
/// <summary> /// Search space is transforms X learners X hyperparameters. /// </summary> private void ComputeSearchSpace(int numTransformLevels, RecipeInference.SuggestedRecipe.SuggestedLearner[] learners, Func <IDataView, TransformInference.Arguments, TransformInference.SuggestedTransform[]> transformInferenceFunction) { _env.AssertValue(_trainData, nameof(_trainData), "Must set training data prior to inferring search space."); var h = _env.Register("ComputeSearchSpace"); using (var ch = h.Start("ComputeSearchSpace")) { _env.Check(IsValidLearnerSet(learners), "Unsupported learner encountered, cannot update search space."); var dataSample = _trainData; var inferenceArgs = new TransformInference.Arguments { EstimatedSampleFraction = 1.0, ExcludeFeaturesConcatTransforms = true }; // Initialize structure for mapping columns back to specific transforms var dependencyMapping = new DependencyMap { { 0, AutoMlUtils.ComputeColumnResponsibilities(dataSample, new TransformInference.SuggestedTransform[0]) } }; // Get suggested transforms for all levels. Defines another part of search space. var transformsList = new List <TransformInference.SuggestedTransform>(); for (int i = 0; i < numTransformLevels; i++) { // Update level for transforms inferenceArgs.Level = i + 1; // Infer transforms using experts var levelTransforms = transformInferenceFunction(dataSample, inferenceArgs); // If no more transforms to apply, dataSample won't change. So end loop. if (levelTransforms.Length == 0) { break; } // Make sure we don't overflow our bitmask if (levelTransforms.Max(t => t.AtomicGroupId) > 64) { break; } // Level-up atomic group id offset. inferenceArgs.AtomicIdOffset = levelTransforms.Max(t => t.AtomicGroupId) + 1; // Apply transforms to dataview for this level. dataSample = AutoMlUtils.ApplyTransformSet(_env, dataSample, levelTransforms); // Keep list of which transforms can be responsible for which output columns dependencyMapping.Add(inferenceArgs.Level, AutoMlUtils.ComputeColumnResponsibilities(dataSample, levelTransforms)); transformsList.AddRange(levelTransforms); } var transforms = transformsList.ToArray(); Func <PipelinePattern, long, bool> verifier = AutoMlUtils.ValidationWrapper(transforms, dependencyMapping); // Save state, for resuming learning _availableTransforms = transforms; _availableLearners = learners; _dependencyMapping = dependencyMapping; _transformedData = dataSample; // Update autoML engine to know what the search space looks like AutoMlEngine.SetSpace(_availableTransforms, _availableLearners, verifier, _trainData, _transformedData, _dependencyMapping, Metric.IsMaximizing); ch.Done(); } }
private PipelinePattern[] GetRandomPipelines(int numOfPipelines) { Host.Check(AvailableLearners.All(l => l.PipelineNode != null)); Host.Check(AvailableTransforms.All(t => t.PipelineNode != null)); int atomicGroupLimit = AvailableTransforms.Select(t => t.AtomicGroupId) .DefaultIfEmpty(-1).Max() + 1; var pipelines = new List <PipelinePattern>(); int collisions = 0; int totalCount = 0; while (pipelines.Count < numOfPipelines) { // Generate random bitmask (set of transform atomic group IDs) long transformsBitMask = Host.Rand.Next((int)Math.Pow(2, atomicGroupLimit)); // Include all "always on" transforms, such as autolabel. transformsBitMask |= AutoMlUtils.IncludeMandatoryTransforms(AvailableTransforms.ToList()); // Get actual learner and transforms for pipeline var selectedLearner = AvailableLearners[Host.Rand.Next(AvailableLearners.Length)]; var selectedTransforms = AvailableTransforms.Where(t => AutoMlUtils.AtomicGroupPresent(transformsBitMask, t.AtomicGroupId)).ToList(); // Randomly change transform sweepable hyperparameter settings selectedTransforms.ForEach(t => RandomlyPerturbSweepableHyperparameters(t.PipelineNode)); // Randomly change learner sweepable hyperparameter settings RandomlyPerturbSweepableHyperparameters(selectedLearner.PipelineNode); // Always include features concat transform selectedTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData, DependencyMapping, selectedTransforms.ToArray(), AvailableTransforms, DataRoles)); // Compute hash key for checking if we've already seen this pipeline. // However, if we keep missing, don't want to get stuck in infinite loop. // Try for a good number of times (for example, numOfPipelines * 4), then just add // all generated pipelines to get us out of rut. string hashKey = GetHashKey(transformsBitMask, selectedLearner); if (collisions < numOfPipelines * 4 && VisitedPipelines.Contains(hashKey)) { collisions++; continue; } VisitedPipelines.Add(hashKey); collisions = 0; totalCount++; // Keep pipeline if valid var pipeline = new PipelinePattern(selectedTransforms.ToArray(), selectedLearner, "", Env); if (!TransformsMaskValidity.ContainsKey(transformsBitMask)) { TransformsMaskValidity.Add(transformsBitMask, PipelineVerifier(pipeline, transformsBitMask)); } if (TransformsMaskValidity[transformsBitMask]) { pipelines.Add(pipeline); } // Only invalid pipelines available, stuck in loop. // Break out and return no pipelines. if (totalCount > numOfPipelines * 10) { break; } } return(pipelines.ToArray()); }
public bool CheckEntryPointStateMatchesParamValues() => AutoMlUtils.CheckEntryPointStateMatchesParamValues(_entryPointObj, SweepParams);
public bool UpdateProperties() => AutoMlUtils.UpdateProperties(_entryPointObj, SweepParams);