private string FindMetric(string userMetric, out bool maximizing)
{
StringBuilder sb = new StringBuilder();
var evaluators = ComponentCatalog.GetAllDerivedClasses(typeof(IMamlEvaluator), typeof(SignatureMamlEvaluator));
foreach (var evalInfo in evaluators)
{
var args = evalInfo.CreateArguments();
var eval = (IEvaluator)evalInfo.CreateInstance(_host, args, new object[0]);
sb.AppendFormat("Metrics for {0}:", evalInfo.UserName);
sb.AppendLine();
foreach (var metric in eval.GetOverallMetricColumns())
{
string result;
if (metric.MetricTarget != MetricColumn.Objective.Info &&
(result = metric.GetNameMatch(userMetric)) != null)
{
maximizing = metric.MetricTarget == MetricColumn.Objective.Maximize;
return(result);
}
sb.AppendFormat("{0} ({1})", metric.LoadName, metric.Name);
sb.AppendLine();
if (metric.CanBeWeighted)
{
sb.AppendFormat("Weighted{0} ({1})", metric.LoadName, metric.Name);
sb.AppendLine();
}
}
}
throw _host.Except("Requested metric '{0}' does not exist. Options are:\n{1}", userMetric, sb.ToString());
}
protected void GenerateMethodSignature(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string parent, string parentType, string parentValue, ref string linePrefix, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
w.WriteLine(linePrefix);
linePrefix = ",";
if (IsTrainer(sigType))
{
w.WriteLine("[DataLabInputPort(FriendlyName = \"Untrained model\", DisplayName = \"Untrained model\", IsOptional = false, DataTypes = WellKnownDataTypeIds.ILearnerDotNet, Description = \"An untrained model\")]");
GenerateParameter(w, "ILearner", arg.LongName + argSuffix);
return;
}
var typeName = EnumName(arg, sigType);
GenerateDataLabParameterAttribute(w, arg.LongName, false, arg.LongName, arg.DefaultValue != null ? typeName + "." + arg.DefaultValue : null, arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, typeName, arg.LongName + argSuffix);
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateMethodSignature(w, a, arg.LongName, typeName, info.LoadNames[0], ref linePrefix, argSuffix + info.LoadNames[0]);
}
}
}
else
{
w.WriteLine(linePrefix);
linePrefix = ",";
if (IsColumnType(arg))
{
GenerateDataLabParameterAttribute(w, arg.LongName, false, arg.LongName, null, arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, "string", arg.LongName + argSuffix);
}
else
{
GenerateDataLabParameterAttribute(w, arg.LongName, IsOptional(arg), arg.LongName, Stringify(arg.DefaultValue), arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, GetCSharpTypeName(arg.ItemType), arg.LongName + argSuffix);
}
}
}
private void GenerateImplCall(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string parent, string parentType, string parentValue, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
if (IsTrainer(sigType))
{
return;
}
var typeName = EnumName(arg, sigType);
GenerateImplCall(w, typeName, arg.LongName + argSuffix);
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateImplCall(w, a, arg.LongName, typeName, info.LoadNames[0], argSuffix + info.LoadNames[0]);
}
}
}
else
{
if (IsColumnType(arg) || IsStringColumnType(arg))
{
string name = arg.LongName + argSuffix;
if (arg.IsCollection)
{
w.WriteLine("builder.{0} = ((string)parameters[\"{1}\"]).Split('|');", Capitalize(name), name);
}
else
{
w.WriteLine("builder.{0} = (string)parameters[\"{1}\"];", Capitalize(name), name);
}
}
else
{
GenerateImplCall(w, GetCSharpTypeName(arg.ItemType), arg.LongName + argSuffix);
}
}
}
private void GenerateMethodSignature(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string parent, string parentType, string parentValue, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
if (IsTrainer(sigType))
{
return;
}
var typeName = EnumName(arg, sigType);
GenerateParameterAttribute(w, arg.LongName, arg.DefaultValue != null ? typeName + "." + arg.DefaultValue : null, arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, typeName, arg.LongName + argSuffix);
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateMethodSignature(w, a, arg.LongName, typeName, info.LoadNames[0], argSuffix + info.LoadNames[0]);
}
}
}
else
{
if (IsColumnType(arg))
{
GenerateParameterAttribute(w, arg.LongName, null, arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, "string", arg.LongName + argSuffix);
}
else
{
GenerateParameterAttribute(w, arg.LongName, Stringify(arg.DefaultValue), arg.HelpText, parent, parentType, parentValue);
GenerateParameter(w, GetCSharpTypeName(arg.ItemType), arg.LongName + argSuffix);
}
}
}
/// <summary>
/// Generates private fields and public properties for all the fields in the arguments.
/// Recursively generate fields and properties for subcomponents.
/// </summary>
protected void GenerateFieldsOrProperties(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix,
Action <IndentingTextWriter, string, string, string, bool, string> oneFieldGenerator)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
if (IsTrainer(sigType))
{
oneFieldGenerator(w, "Tuple<string, string>", arg.LongName + argSuffix, "", false, arg.HelpText);
return;
}
var typeName = EnumName(arg, sigType);
string defVal = arg.DefaultValue != null?string.Format(" = {0}.{1}", typeName, arg.DefaultValue) : "";
oneFieldGenerator(w, typeName, arg.LongName + argSuffix, defVal, arg.ItemType == typeof(bool),
arg.HelpText);
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateFieldsOrProperties(w, a, argSuffix + info.LoadNames[0], oneFieldGenerator);
}
}
}
else
{
object val = Stringify(arg.DefaultValue);
string defVal = val == null ? "" : string.Format(" = {0}", val);
var typeName = IsColumnType(arg) ? "string[]" : IsStringColumnType(arg) ? GetCSharpTypeName(arg.Field.FieldType) : GetCSharpTypeName(arg.ItemType);
oneFieldGenerator(w, typeName, arg.LongName + argSuffix, defVal, arg.ItemType == typeof(bool), arg.HelpText);
}
}
protected void GenerateImplCall(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
if (IsTrainer(sigType))
{
w.WriteLine("builder.{0} = {1};", Capitalize(arg.LongName + argSuffix), arg.LongName + argSuffix);
return;
}
w.WriteLine("builder.{0} = {1};", Capitalize(arg.LongName + argSuffix), arg.LongName + argSuffix);
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateImplCall(w, a, argSuffix + info.LoadNames[0]);
}
}
}
else
{
if (IsColumnType(arg))
{
w.WriteLine("builder.{0} = {1}.Split('|');", Capitalize(arg.LongName + argSuffix), arg.LongName + argSuffix);
}
else
{
w.WriteLine("builder.{0} = {1}{2};", Capitalize(arg.LongName + argSuffix), CastIfNeeded(arg), arg.LongName + argSuffix);
}
}
}
public static List <Sweep> GenerateCandidates(IHostEnvironment env, string dataFile, string schemaDefinitionFile)
{
var patterns = new List <Sweep>();
string loaderSettings;
Type predictorType;
TransformInference.InferenceResult inferenceResult;
// Get the initial recipes for this data.
RecipeInference.SuggestedRecipe[] recipes = RecipeInference.InferRecipesFromData(env, dataFile, schemaDefinitionFile, out predictorType, out loaderSettings, out inferenceResult);
//get all the trainers for this task, and generate the initial set of candidates.
// Exclude the hidden learners, and the metalinear learners.
var trainers = ComponentCatalog.GetAllDerivedClasses(typeof(ITrainer), predictorType).Where(cls => !cls.IsHidden);
if (!string.IsNullOrEmpty(loaderSettings))
{
StringBuilder sb = new StringBuilder();
CmdQuoter.QuoteValue(loaderSettings, sb, true);
loaderSettings = sb.ToString();
}
string loader = $" loader=TextLoader{loaderSettings}";
// REVIEW: there are more learners than recipes atm.
// Flip looping through recipes, then through learners if the cardinality changes.
foreach (ComponentCatalog.LoadableClassInfo cl in trainers)
{
string learnerSettings;
TrainerSweeper trainerSweeper = new TrainerSweeper();
trainerSweeper.Parameters.AddRange(RecipeInference.GetLearnerSettingsAndSweepParams(env, cl, out learnerSettings));
foreach (var recipe in recipes)
{
RecipeInference.SuggestedRecipe.SuggestedLearner learner = new RecipeInference.SuggestedRecipe.SuggestedLearner
{
LoadableClassInfo = cl,
Settings = learnerSettings
};
Pattern pattern = new Pattern(recipe.Transforms, learner, loader);
Sweep sweep = new Sweep(pattern, trainerSweeper);
patterns.Add(sweep);
}
}
return(patterns);
}
protected void GenerateSummaryComment(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText);
if (IsTrainer(sigType))
{
return;
}
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateSummaryComment(w, a, argSuffix + info.LoadNames[0]);
}
}
}
else
{
GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText);
}
}
private void ShowComponents(IndentingTextWriter writer)
{
Type typeSig;
Type typeRes;
string kind;
if (_kind == null)
{
// Show commands.
typeSig = typeof(SignatureCommand);
typeRes = typeof(ICommand);
kind = "Command";
writer.WriteLine("Available commands:");
}
else
{
kind = _kind.ToLowerInvariant();
var sigs = ComponentCatalog.GetAllSignatureTypes();
typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).ToLowerInvariant() == kind);
if (typeSig == null)
{
typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).StartsWithInvariantCultureIgnoreCase(kind));
if (typeSig == null)
{
writer.WriteLine("Couldn't find kind '{0}'", kind);
ListKinds(writer);
return;
}
}
typeRes = typeof(object);
writer.WriteLine("Available components for kind '{0}':", ComponentCatalog.SignatureToString(typeSig));
}
var infos = ComponentCatalog.GetAllDerivedClasses(typeRes, typeSig)
.Where(x => !x.IsHidden)
.OrderBy(x => x.LoadNames[0].ToLowerInvariant());
using (writer.Nest())
{
var components = new List <Component>();
foreach (var info in infos)
{
_env.Assert(info.LoadNames.Count > 0);
writer.Write("{0}", info.LoadNames[0]);
if (!string.IsNullOrWhiteSpace(info.UserName))
{
writer.Write(": {0}", info.UserName);
}
writer.WriteLine();
using (writer.Nest())
ShowAliases(writer, info.LoadNames);
components.Add(new Component(kind, info, info.CreateArguments()));
}
if (components.Count > 0)
{
Serialize(components);
}
}
}
protected void GenerateImplBody(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (IsTrainer(sigType))
{
if (arg.IsCollection)
{
w.WriteLine("args{0}.{1} = new[] {{ new SubComponent<{2}, {3}>({4}.Item1, {4}.Item2) }};",
argSuffix, arg.LongName, GetCSharpTypeName(baseType), GetCSharpTypeName(sigType),
arg.LongName + argSuffix);
}
else
{
w.WriteLine("args{0}.{1} = new SubComponent<{2}, {3}>({4}.Item1, {4}.Item2);", argSuffix,
arg.LongName, GetCSharpTypeName(baseType), GetCSharpTypeName(sigType),
arg.LongName + argSuffix);
}
return;
}
if (sigType == typeof(SignatureDataLoader))
{
return;
}
var typeName = EnumName(arg, sigType);
w.WriteLine("switch ({0})", arg.LongName + argSuffix);
w.WriteLine("{");
using (w.Nest())
{
if (arg.NullName != null)
{
w.WriteLine("case {0}.None:", typeName);
using (w.Nest())
{
w.WriteLine("args{0}.{1} = null;", argSuffix, arg.LongName);
w.WriteLine("break;");
}
}
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
w.WriteLine("case {0}.{1}:", typeName, info.LoadNames[0]);
using (w.Nest())
{
if (info.ArgType != null)
{
var newArgSuffix = argSuffix + info.LoadNames[0];
w.WriteLine("var args{0} = new {1}();", newArgSuffix, GetCSharpTypeName(info.ArgType));
w.WriteLine("var defs{0} = new {1}();", newArgSuffix, GetCSharpTypeName(info.ArgType));
var args = info.CreateArguments();
if (args != null)
{
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateImplBody(w, a, newArgSuffix);
}
}
w.WriteLine(
"args{0}.{1} = new {2}(\"{3}\", CmdParser.GetSettings(args{4}, defs{4}));",
argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType), info.LoadNames[0],
newArgSuffix);
}
else
{
w.WriteLine("args{0}.{1} = new {2}(\"{3}\");", argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType), info.LoadNames[0]);
}
w.WriteLine("break;");
}
}
}
w.WriteLine("}");
}
else if (arg.IsCollection)
{
if (IsColumnType(arg))
{
w.WriteLine("args{0}.{1} = {1}.Select({2}.Parse).ToArray();", argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType));
}
else if (IsStringColumnType(arg))
{
w.WriteLine("args{0}.{1} = {2};", argSuffix, arg.LongName, arg.LongName + argSuffix);
}
else
{
w.WriteLine("args{0}.{1} = new[] {{ {2} }};", argSuffix, arg.LongName, arg.LongName + argSuffix);
}
}
else
{
w.WriteLine("args{0}.{1} = {2};", argSuffix, arg.LongName, arg.LongName + argSuffix);
}
}
/// <summary>
/// Generate enums for subcomponents. Uses ReflectionUtils to filter only the subcomponents that match the base type and the signature.
/// </summary>
/// <param name="w"></param>
/// <param name="arg"></param>
/// <param name="seen"></param>
protected void GenerateEnums(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, HashSet <Tuple <Type, Type> > seen)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
var moreEnums = new List <CmdParser.ArgInfo.Arg>();
if (arg.IsHidden || !arg.IsSubComponentItemType)
{
return;
}
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
var key = new Tuple <Type, Type>(baseType, sigType);
if (seen.Contains(key) || IsTrainer(sigType) || sigType == typeof(SignatureDataLoader))
{
return;
}
seen.Add(key);
var typeName = EnumName(arg, sigType);
w.WriteLine("/// <summary> Available choices for {0} </summary>", sigType);
w.WriteLine("public enum {0}", typeName);
w.Write("{");
using (w.Nest())
{
var pre = "";
if (arg.NullName != null)
{
w.WriteLine();
GenerateEnumValue(w, null);
pre = ",";
}
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
w.WriteLine(pre);
if (pre != "")
{
w.WriteLine();
}
pre = ",";
GenerateEnumValue(w, info);
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
moreEnums.AddRange(argInfo.Args);
}
w.WriteLine();
}
w.WriteLine("}");
w.WriteLine();
foreach (var argument in moreEnums)
{
GenerateEnums(w, argument, seen);
}
}