protected string GetHashKey(long transformsBitMask, RecipeInference.SuggestedRecipe.SuggestedLearner learner)
{
var learnerName = learner.ToString();
Host.Check(!string.IsNullOrEmpty(learnerName));
return($"{learnerName}+{transformsBitMask}");
}
private void SampleHyperparameters(RecipeInference.SuggestedRecipe.SuggestedLearner learner, PipelinePattern[] history)
{
// If first time optimizing hyperparams, create new hyperparameter sweeper.
if (!_hyperSweepers.ContainsKey(learner.LearnerName))
{
var sps = AutoMlUtils.ConvertToComponentFactories(learner.PipelineNode.SweepParams);
if (sps.Length > 0)
{
_hyperSweepers[learner.LearnerName] = new KdoSweeper(Env,
new KdoSweeper.Arguments
{
SweptParameters = sps,
NumberInitialPopulation = Math.Max(_remainingThirdStageTrials, 2)
});
}
else
{
_hyperSweepers[learner.LearnerName] = new FalseSweeper();
}
}
var sweeper = _hyperSweepers[learner.LearnerName];
var historyToUse = history.Where(p => p.Learner.LearnerName == learner.LearnerName).ToArray();
if (_currentStage == (int)Stages.Third)
{
_remainingThirdStageTrials--;
historyToUse = new PipelinePattern[0];
if (_remainingThirdStageTrials < 1)
{
_currentStage++;
}
}
SampleHyperparameters(learner, sweeper, IsMaximizingMetric, historyToUse);
}
internal Pattern(TransformInference.SuggestedTransform[] transforms,
RecipeInference.SuggestedRecipe.SuggestedLearner learner,
string loader)
{
Transforms = transforms;
Learner = learner;
Loader = loader;
}
public static List <Sweep> GenerateCandidates(IHostEnvironment env, string dataFile, string schemaDefinitionFile)
{
var patterns = new List <Sweep>();
string loaderSettings;
Type predictorType;
TransformInference.InferenceResult inferenceResult;
// Get the initial recipes for this data.
RecipeInference.SuggestedRecipe[] recipes = RecipeInference.InferRecipesFromData(env, dataFile, schemaDefinitionFile, out predictorType, out loaderSettings, out inferenceResult);
//get all the trainers for this task, and generate the initial set of candidates.
// Exclude the hidden learners, and the metalinear learners.
var trainers = env.ComponentCatalog.GetAllDerivedClasses(typeof(ITrainer), predictorType).Where(cls => !cls.IsHidden);
if (!string.IsNullOrEmpty(loaderSettings))
{
StringBuilder sb = new StringBuilder();
CmdQuoter.QuoteValue(loaderSettings, sb, true);
loaderSettings = sb.ToString();
}
string loader = $" loader=TextLoader{loaderSettings}";
// REVIEW: there are more learners than recipes atm.
// Flip looping through recipes, then through learners if the cardinality changes.
foreach (ComponentCatalog.LoadableClassInfo cl in trainers)
{
string learnerSettings;
TrainerSweeper trainerSweeper = new TrainerSweeper();
trainerSweeper.Parameters.AddRange(RecipeInference.GetLearnerSettingsAndSweepParams(env, cl, out learnerSettings));
foreach (var recipe in recipes)
{
RecipeInference.SuggestedRecipe.SuggestedLearner learner = new RecipeInference.SuggestedRecipe.SuggestedLearner
{
LoadableClassInfo = cl,
Settings = learnerSettings
};
Pattern pattern = new Pattern(recipe.Transforms, learner, loader);
Sweep sweep = new Sweep(pattern, trainerSweeper);
patterns.Add(sweep);
}
}
return(patterns);
}
public PipelinePattern(TransformInference.SuggestedTransform[] transforms,
RecipeInference.SuggestedRecipe.SuggestedLearner learner,
string loaderSettings, IHostEnvironment env, PipelineSweeperRunSummary summary = null)
{
// Make sure internal pipeline nodes and sweep params are cloned, not shared.
// Cloning the transforms and learner rather than assigning outright
// ensures that this will be the case. Doing this here allows us to not
// worry about changing hyperparameter values in candidate pipelines
// possibly overwritting other pipelines.
Transforms = transforms.Select(t => t.Clone()).ToArray();
Learner = learner.Clone();
LoaderSettings = loaderSettings;
_env = env;
PerformanceSummary = summary;
UniqueId = Guid.NewGuid();
}
private static ParameterSet ConvertToParameterSet(TlcModule.SweepableParamAttribute[] hps,
RecipeInference.SuggestedRecipe.SuggestedLearner learner)
{
if (learner.PipelineNode.HyperSweeperParamSet != null)
{
return(learner.PipelineNode.HyperSweeperParamSet);
}
var paramValues = new IParameterValue[hps.Length];
if (hps.Any(p => p.RawValue == null))
{
PopulateSweepableParams(learner);
}
for (int i = 0; i < hps.Length; i++)
{
Contracts.CheckValue(hps[i].RawValue, nameof(TlcModule.SweepableParamAttribute.RawValue));
switch (hps[i])
{
case TlcModule.SweepableDiscreteParamAttribute dp:
var learnerVal =
learner.PipelineNode.GetPropertyValueByName(dp.Name, (IComparable)dp.Options[0]);
var optionIndex = (int)(dp.RawValue ?? dp.IndexOf(learnerVal));
paramValues[i] = new StringParameterValue(dp.Name, dp.Options[optionIndex].ToString());
break;
case TlcModule.SweepableFloatParamAttribute fp:
paramValues[i] =
new FloatParameterValue(fp.Name,
(float)(fp.RawValue ?? learner.PipelineNode.GetPropertyValueByName(fp.Name, 0f)));
break;
case TlcModule.SweepableLongParamAttribute lp:
paramValues[i] =
new LongParameterValue(lp.Name,
(long)(lp.RawValue ?? learner.PipelineNode.GetPropertyValueByName(lp.Name, 0L)));
break;
}
}
learner.PipelineNode.HyperSweeperParamSet = new ParameterSet(paramValues);
return(learner.PipelineNode.HyperSweeperParamSet);
}
/// <summary>
/// Updates properties of entryPointObj instance based on the values in sweepParams
/// </summary>
public static void PopulateSweepableParams(RecipeInference.SuggestedRecipe.SuggestedLearner learner)
{
foreach (var param in learner.PipelineNode.SweepParams)
{
if (param is TlcModule.SweepableDiscreteParamAttribute dp)
{
var learnerVal = learner.PipelineNode.GetPropertyValueByName(dp.Name, (IComparable)dp.Options[0]);
param.RawValue = dp.IndexOf(learnerVal);
}
else if (param is TlcModule.SweepableFloatParamAttribute fp)
{
param.RawValue = learner.PipelineNode.GetPropertyValueByName(fp.Name, 0f);
}
else if (param is TlcModule.SweepableLongParamAttribute lp)
{
param.RawValue = learner.PipelineNode.GetPropertyValueByName(lp.Name, 0L);
}
}
}
protected void SampleHyperparameters(RecipeInference.SuggestedRecipe.SuggestedLearner learner, ISweeper sweeper,
bool isMaximizingMetric, PipelinePattern[] history)
{
// Make sure there are hyperparameters to sweep over.
var hyperParams = learner.PipelineNode.SweepParams;
if (hyperParams.Length == 0)
{
return;
}
// Get new set of hyperparameter values.
var proposedParamSet = sweeper.ProposeSweeps(1, AutoMlUtils.ConvertToRunResults(history, isMaximizingMetric)).First();
Env.Assert(proposedParamSet != null && proposedParamSet.All(ps => hyperParams.Any(hp => hp.Name == ps.Name)));
// Associate proposed param set with learner, so that smart hyperparam
// sweepers (like KDO) can map them back.
learner.PipelineNode.HyperSweeperParamSet = proposedParamSet;
var generatorSet = hyperParams.Select(AutoMlUtils.ToIValueGenerator).ToArray();
var values = SweeperProbabilityUtils.ParameterSetAsFloatArray(Host, generatorSet, proposedParamSet, false);
// Update hyperparameters.
for (int i = 0; i < hyperParams.Length; i++)
{
if (hyperParams[i] is TlcModule.SweepableDiscreteParamAttribute dp)
{
hyperParams[i].RawValue = (int)values[i];
}
else
{
hyperParams[i].RawValue = values[i];
}
}
}
private PipelinePattern[] NextCandidates(PipelinePattern[] history, int numCandidates,
bool defaultHyperParams = false, bool uniformRandomTransforms = false)
{
const int maxNumberAttempts = 10;
double[] learnerWeights = LearnerHistoryToWeights(history, IsMaximizingMetric);
var candidates = new List <PipelinePattern>();
var sampledLearners = new RecipeInference.SuggestedRecipe.SuggestedLearner[numCandidates];
if (_currentStage == (int)Stages.Second || _currentStage == (int)Stages.Third)
{
// Select remaining learners in round-robin fashion.
for (int i = 0; i < numCandidates; i++)
{
sampledLearners[i] = AvailableLearners[i % AvailableLearners.Length].Clone();
}
}
else
{
// Select learners, based on weights.
var indices = ProbUtils.SampleCategoricalDistribution(numCandidates, learnerWeights);
foreach (var item in indices.Select((idx, i) => new { idx, i }))
{
sampledLearners[item.i] = AvailableLearners[item.idx].Clone();
}
}
// Select hyperparameters and transforms based on learner and history.
foreach (var learner in sampledLearners)
{
PipelinePattern pipeline;
int count = 0;
bool valid;
string hashKey;
if (!defaultHyperParams)
{
SampleHyperparameters(learner, history);
}
else
{
AutoMlUtils.PopulateSweepableParams(learner);
}
do
{ // Make sure transforms set is valid and have not seen pipeline before.
// Repeat until passes or runs out of chances.
pipeline = new PipelinePattern(
SampleTransforms(learner, history, out var transformsBitMask, uniformRandomTransforms),
learner, "", Env);
hashKey = GetHashKey(transformsBitMask, learner);
valid = PipelineVerifier(pipeline, transformsBitMask) && !VisitedPipelines.Contains(hashKey);
count++;
} while (!valid && count <= maxNumberAttempts);
// If maxed out chances and at second stage, move onto next stage.
if (count >= maxNumberAttempts && _currentStage == (int)Stages.Second)
{
_currentStage++;
}
// Keep only valid pipelines.
if (valid)
{
VisitedPipelines.Add(hashKey);
candidates.Add(pipeline);
}
}
return(candidates.ToArray());
}
private TransformInference.SuggestedTransform[] SampleTransforms(RecipeInference.SuggestedRecipe.SuggestedLearner learner,
PipelinePattern[] history, out long transformsBitMask, bool uniformRandomSampling = false)
{
var sampledTransforms =
new List <TransformInference.SuggestedTransform>(
AutoMlUtils.GetMandatoryTransforms(AvailableTransforms));
var remainingAvailableTransforms =
AvailableTransforms.Where(t => !sampledTransforms.Any(t.Equals)).ToArray();
var mask = AutoMlUtils.TransformsToBitmask(sampledTransforms.ToArray());
foreach (var transform in remainingAvailableTransforms)
{
// Add pseudo-mass to encourage sampling of untried transforms.
double maxWeight = history.Length > 0 ? history.Max(w => w.PerformanceSummary.MetricValue) : 0d;
double allWeight = Math.Max(maxWeight, 1d);
double learnerWeight = Math.Max(maxWeight, 1d);
int allCounts = 1;
int learnerCounts = 1;
// Add mass according to performance.
foreach (var pipeline in history)
{
if (pipeline.Transforms.Any(transform.Equals))
{
allWeight +=
AutoMlUtils.ProcessWeight(pipeline.PerformanceSummary.MetricValue,
maxWeight, IsMaximizingMetric);
allCounts++;
if (pipeline.Learner.LearnerName == learner.LearnerName)
{
learnerWeight += pipeline.PerformanceSummary.MetricValue;
learnerCounts++;
}
}
}
// Take average mass as weight, and take convex combination of
// learner-specific weight and unconditioned weight.
allWeight /= allCounts > 0 ? allCounts : 1;
learnerWeight /= learnerCounts > 0 ? learnerCounts : 1;
var lambda = MathUtils.Sigmoid(learnerCounts - 3);
var combinedWeight = uniformRandomSampling ?
0.5 : lambda * learnerWeight + (1 - lambda) * allWeight;
// Sample transform according to combined weight.
if (ProbUtils.SampleUniform() <= combinedWeight / maxWeight)
{
mask |= 1L << transform.AtomicGroupId;
}
}
// Add all chosen transforms.
sampledTransforms.AddRange(remainingAvailableTransforms.Where(t =>
AutoMlUtils.AtomicGroupPresent(mask, t.AtomicGroupId)));
// Add final features concat transform. NOTE: computed bitmask should always
// exclude the final features concat. If we forget to exclude that one, will
// cause an error in verification, since it isn't included in the original
// dependency mapping (i.e., its level isn't in the dictionary).
sampledTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData,
DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms, DataRoles));
transformsBitMask = mask;
return(sampledTransforms.ToArray());
}
public static IRunResult ConvertToRunResult(RecipeInference.SuggestedRecipe.SuggestedLearner learner, PipelineSweeperRunSummary rs, bool isMetricMaximizing)
{
return(new RunResult(ConvertToParameterSet(learner.PipelineNode.SweepParams, learner), rs.MetricValue, isMetricMaximizing));
}
