protected void GenerateModuleType(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
string cat;
if (component.IsOfType(typeof(SignatureBinaryClassifierTrainer)))
{
cat = "BinaryClassifier";
}
else if (component.IsOfType(typeof(SignatureMultiClassClassifierTrainer)))
{
cat = "MultiClassClassifier";
}
else if (component.IsOfType(typeof(SignatureRegressorTrainer)))
{
cat = "Regression";
}
else if (component.IsOfType(typeof(SignatureAnomalyDetectorTrainer)))
{
cat = "AnomalyDetector";
}
else
{
cat = "None";
}
w.WriteLine("[DataLabModuleType(Type = ModuleType.{0})]", cat);
}
protected override void GenerateEnumValue(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo info)
{
var name = info != null ? info.LoadNames[0] : "None";
w.WriteLine("/// <summary> {0} option </summary>", name);
w.Write("{0}", name);
}
public CrossValidationCommand(IHostEnvironment env, Arguments args)
: base(env, args, RegistrationName)
{
Host.CheckUserArg(Args.NumFolds >= 2, nameof(Args.NumFolds), "Number of folds must be greater than or equal to 2.");
_info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile);
Utils.CheckOptionalUserDirectory(Args.SummaryFilename, nameof(Args.SummaryFilename));
Utils.CheckOptionalUserDirectory(Args.OutputDataFile, nameof(Args.OutputDataFile));
}
protected override void GenerateEnumValue(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo info)
{
var userName = info != null ? info.UserName : "******";
var name = info != null ? info.LoadNames[0] : "None";
w.WriteLine("[ItemInfo(FriendlyName = \"{0}\", DisplayValue = \"{1}\")]", userName,
userName);
w.Write("{0}", name);
}
/// <summary>
/// Determine the default scorer for a schema bound mapper. This looks for text-valued ScoreColumnKind
/// metadata on the first column of the mapper. If that text is found and maps to a scorer loadable class,
/// that component is used. Otherwise, the GenericScorer is used.
/// </summary>
/// <param name="environment">The host environment.</param>.
/// <param name="mapper">The schema bound mapper to get the default scorer.</param>.
/// <param name="suffix">An optional suffix to append to the default column names.</param>
public static TScorerFactory GetScorerComponent(
IHostEnvironment environment,
ISchemaBoundMapper mapper,
string suffix = null)
{
Contracts.CheckValue(environment, nameof(environment));
Contracts.AssertValue(mapper);
ComponentCatalog.LoadableClassInfo info = null;
ReadOnlyMemory <char> scoreKind = default;
if (mapper.OutputSchema.Count > 0 &&
mapper.OutputSchema.TryGetMetadata(TextType.Instance, MetadataUtils.Kinds.ScoreColumnKind, 0, ref scoreKind) &&
!scoreKind.IsEmpty)
{
var loadName = scoreKind.ToString();
info = environment.ComponentCatalog.GetLoadableClassInfo <SignatureDataScorer>(loadName);
if (info == null || !typeof(IDataScorerTransform).IsAssignableFrom(info.Type))
{
info = null;
}
}
Func <IHostEnvironment, IDataView, ISchemaBoundMapper, RoleMappedSchema, IDataScorerTransform> factoryFunc;
if (info == null)
{
factoryFunc = (env, data, innerMapper, trainSchema) =>
new GenericScorer(
env,
new GenericScorer.Arguments()
{
Suffix = suffix
},
data,
innerMapper,
trainSchema);
}
else
{
factoryFunc = (env, data, innerMapper, trainSchema) =>
{
object args = info.CreateArguments();
if (args is ScorerArgumentsBase scorerArgs)
{
scorerArgs.Suffix = suffix;
}
return((IDataScorerTransform)info.CreateInstance(
env,
args,
new object[] { data, innerMapper, trainSchema }));
};
}
return(ComponentFactoryUtils.CreateFromFunction(factoryFunc));
}
protected void GenerateEnums(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var seen = new HashSet <Tuple <Type, Type> >();
foreach (var arg in argumentInfo.Args)
{
GenerateEnums(w, arg, seen);
}
}
public Component(string kind, ComponentCatalog.LoadableClassInfo info, object args)
{
Contracts.AssertNonEmpty(kind);
Contracts.AssertValue(info);
Contracts.AssertValueOrNull(args);
Kind = kind;
Info = info;
Args = args;
}
public TrainTestCommand(IHostEnvironment env, Arguments args)
: base(env, args, nameof(TrainTestCommand))
{
Utils.CheckOptionalUserDirectory(args.SummaryFilename, nameof(args.SummaryFilename));
Utils.CheckOptionalUserDirectory(args.OutputDataFile, nameof(args.OutputDataFile));
_info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile);
if (string.IsNullOrWhiteSpace(args.TestFile))
{
throw Host.ExceptUserArg(nameof(args.TestFile), "Test file must be defined.");
}
}
protected void GenerateImplFields(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component,
Action <IndentingTextWriter, CmdParser.ArgInfo.Arg> fieldGenerator)
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
fieldGenerator(w, arg);
}
}
public TrainCommand(IHostEnvironment env, Arguments args)
: base(env, args, nameof(TrainCommand))
{
Host.CheckNonWhiteSpace(args.OutputModelFile, nameof(args.OutputModelFile));
_info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile);
_trainer = args.Trainer;
_labelColumn = args.LabelColumn;
_featureColumn = args.FeatureColumn;
_groupColumn = args.GroupColumn;
_weightColumn = args.WeightColumn;
_nameColumn = args.NameColumn;
}
private void GenerateFile(ComponentCatalog.LoadableClassInfo info, string filename, Dictionary <Type, GeneratorBase> mapping, bool generateEnums)
{
using (var sw = new StreamWriter(filename))
{
var writer = IndentingTextWriter.Wrap(sw, " ");
foreach (var kvp in mapping)
{
if (info.IsOfType(kvp.Key))
{
kvp.Value.Generate(writer, _modulePrefix, _regenerate, info, generateEnums,
_moduleId ?? Guid.NewGuid().ToString(), _moduleName, _moduleOwner, _moduleVersion, _moduleState,
_moduleType, _moduleDeterminism, _moduleCategory, _exclude, _namespaces);
break;
}
}
}
}
protected override void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType));
w.WriteLine("var defs = new {0}();", GetCSharpTypeName(component.ArgType));
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
GenerateImplBody(w, arg, "");
}
w.WriteLine("return new Tuple<string, string>(\"{0}\", CmdParser.GetSettings(args, defs));", component.LoadNames[0]);
}
w.WriteLine("}");
}
protected override void GenerateMethodSignature(IndentingTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("public static Tuple<IDataView, DataTransform> Create{0}{1}(", prefix, component.LoadNames[0]);
using (w.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
w.WriteLine("[DataLabInputPort(FriendlyName = \"IDataView\", DisplayName = \"IDataView\", IsOptional = false, DataTypes = WellKnownDataTypeIds.IDataViewDotNet, Description = \"Input data (IDataView)\")]");
w.Write("IDataView data");
var pre = ",";
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateMethodSignature(w, arg, null, null, null, ref pre, "");
}
w.WriteLine(")");
}
}
private string GenerateCall(ComponentCatalog.LoadableClassInfo component)
{
// The caller needs to ensure that the component has either a constructor or a create method.
Contracts.Assert(component.Constructor != null || component.CreateMethod != null);
string call;
if (component.Constructor != null)
{
var type = GetCSharpTypeName(component.Constructor.DeclaringType);
call = string.Format("new {0}", type);
}
else
{
var type = GetCSharpTypeName(component.CreateMethod.DeclaringType);
var name = component.CreateMethod.Name;
call = string.Format("{0}.{1}", type, name);
}
return(call);
}
protected override void GenerateModuleAttribute(IndentedTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component, string moduleId)
{
if (!string.IsNullOrEmpty(prefix))
{
prefix += " ";
}
w.WriteLine("[DataLabModule(FriendlyName = \"{0}{1}\",", prefix, component.UserName);
using (w.Nest())
{
var desc = component.Summary ?? component.LoadNames[0];
w.WriteLine("Description = \"{0}\",", desc.Replace("\n", "\\n").Replace("\r", "\\r"));
w.WriteLine("IsBlocking = true,");
w.WriteLine("IsDeterministic = true,");
w.WriteLine("Version = \"2.0\",");
w.WriteLine("Owner = \"Microsoft Corporation\",");
w.WriteLine("FamilyId = \"{{{0}}}\",", moduleId.ToUpperInvariant());
w.WriteLine("ReleaseState = States.Alpha)]");
}
}
protected override void GenerateSummaryComment(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("/// <summary>");
var desc = component.Summary ?? component.LoadNames[0];
using (var sr = new StringReader(desc))
{
string line;
while ((line = sr.ReadLine()) != null)
{
w.WriteLine("/// {0}", line);
}
}
w.WriteLine("/// </summary>");
GenerateParameterComment(w, "data", "The data");
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateSummaryComment(w, arg, "");
}
}
private void GenerateFile(IChannel ch, ComponentCatalog.LoadableClassInfo info)
{
_host.AssertValue(ch);
ch.AssertValue(info);
string name = info.LoadNames[0];
if (!info.IsOfType(typeof(SignatureTrainer)) && !info.IsOfType(typeof(SignatureDataTransform)))
{
ch.Warning("No generator available for {0}.", name);
return;
}
if (info.Constructor == null && info.CreateMethod == null)
{
ch.Warning("No construction method available for {0}.", name);
return;
}
if (_generateModule)
{
var entryPointFile = _modulePrefix + name + "EntryPoint.cs";
if (_generateModuleInstance)
{
GenerateFile(info, entryPointFile, ModuleInstanceEntryPointGeneratorMapping);
}
else
{
GenerateFile(info, entryPointFile, EntryPointGeneratorMapping);
}
}
var implFile = _modulePrefix + name + ".cs";
GenerateFile(info, implFile, ImplGeneratorMapping);
}
protected override void GenerateImplBody(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
if (component.ArgType == null)
{
var call = GenerateCall(component);
w.WriteLine("return {0}(env, data);", call);
}
else
{
w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType));
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplBody(w, arg, "");
}
var call = GenerateCall(component);
w.WriteLine("return {0}(args, env, data);", call);
}
}
w.WriteLine("}");
}
public static List <string> GetLearnerSettingsAndSweepParams(IHostEnvironment env, ComponentCatalog.LoadableClassInfo cl, out string settings)
{
List <string> sweepParams = new List <string>();
var ci = cl.Constructor?.GetParameters();
if (ci == null)
{
settings = "";
return(sweepParams);
}
var suggestedSweepsParser = new SuggestedSweepsParser();
StringBuilder learnerSettings = new StringBuilder();
foreach (var prop in ci)
{
var fieldInfo = prop.ParameterType?.GetFields(BindingFlags.Public | BindingFlags.Instance);
foreach (var field in fieldInfo)
{
TGUIAttribute[] tgui =
field.GetCustomAttributes(typeof(TGUIAttribute), true) as TGUIAttribute[];
if (tgui == null)
{
continue;
}
foreach (var attr in tgui)
{
if (attr.SuggestedSweeps != null)
{
// Build the learner setting.
learnerSettings.Append($" {field.Name}=${field.Name}$");
// Build the sweeper.
suggestedSweepsParser.TryParseParameter(attr.SuggestedSweeps, field.FieldType, field.Name, out var sweepValues, out var error);
sweepParams.Add(sweepValues?.ToStringParameter(env));
}
}
}
}
settings = learnerSettings.ToString();
return(sweepParams);
}
/// <summary>
/// Generate the module and its implementation.
/// </summary>
/// <param name="writer">The writer.</param>
/// <param name="prefix">The module prefix.</param>
/// <param name="regenerate">The command string used to generate.</param>
/// <param name="component">The component.</param>
/// <param name="moduleId"></param>
/// <param name="moduleName"></param>
/// <param name="moduleOwner"></param>
/// <param name="moduleVersion"></param>
/// <param name="moduleState"></param>
/// <param name="moduleType"></param>
/// <param name="moduleDeterminism"></param>
/// <param name="moduleCategory"></param>
/// <param name="exclude">The set of parameters to exclude</param>
/// <param name="namespaces">The set of extra namespaces</param>
public void Generate(IndentedTextWriter writer, string prefix, string regenerate, ComponentCatalog.LoadableClassInfo component,
string moduleId, string moduleName, string moduleOwner, string moduleVersion, string moduleState, string moduleType, string moduleDeterminism, string moduleCategory,
HashSet <string> exclude, HashSet <string> namespaces)
{
Contracts.AssertValue(exclude);
Name = moduleName;
Owner = moduleOwner;
Version = moduleVersion;
State = moduleState;
ModuleType = moduleType;
Determinism = moduleDeterminism;
Category = moduleCategory;
Exclude = exclude;
Namespaces = namespaces;
GenerateHeader(writer, regenerate);
using (writer.Nest())
{
GenerateClassName(writer, prefix, component);
using (writer.Nest())
GenerateContent(writer, prefix, component, moduleId);
GenerateFooter(writer);
}
GenerateFooter(writer);
}
protected virtual void GenerateClassName(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine();
var className = prefix + component.LoadNames[0];
w.WriteLine("/// <summary>Module: {0}</summary>", className);
w.WriteLine("public partial class {0}", className);
w.WriteLine("{");
}
protected override void GenerateClassName(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine();
var className = prefix + component.LoadNames[0];
w.WriteLine("/// <summary>Module: {0}</summary>", className);
w.WriteLine("[Obsolete]");
w.WriteLine("public static class {0}EntryPoint", className);
w.WriteLine("{");
}
protected override void GenerateContent(IndentedTextWriter writer, string prefix,
ComponentCatalog.LoadableClassInfo component, string moduleId)
{
writer.WriteLine("[Module(");
_compName = prefix + component.LoadNames[0];
var name = Name ?? PrettyPrintDisplayName(component.LoadNames[0]);
using (writer.Nest())
{
writer.WriteLine("Name = \"{0}\",", name);
writer.WriteLine("FamilyId = \"{0}\",", moduleId);
writer.WriteLine("Owner = \"{0}\",", Owner);
writer.WriteLine("ReleaseVersion = \"{0}\",", Version);
writer.WriteLine("State = ModuleState.{0},", State);
writer.WriteLine("Type = ModuleType.{0},", ModuleType);
writer.WriteLine("Determinism = Determinism.{0},", Determinism);
writer.WriteLine("Category = @\"{0}\")]", Category);
}
writer.WriteLine("[Obsolete]");
writer.WriteLine("public static IModule Create{0}(", _compName);
using (writer.Nest())
{
writer.WriteLine("[Help(Display = @\"Dataset\", ToolTip = @\"Input dataset\")]");
writer.WriteLine("[ModuleInputPort]");
writer.WriteLine("IDataView idataset,");
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateMethodSignature(writer, arg, null, null, null, "");
}
writer.WriteLine("[Help(Display = @\"Results dataset\", ToolTip = @\"Transformed dataset\")]");
writer.WriteLine("[ModuleOutputPort]");
writer.WriteLine("IDataView odataset,");
writer.WriteLine("[Help(Display = @\"{0}\", ToolTip = @\"{0}\")]", name);
writer.WriteLine("[ModuleOutputPort]");
writer.WriteLine("DataTransform otransform,");
writer.WriteLine("[Context]");
writer.WriteLine("IContext context)");
}
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var instance = new {0}Module();", _compName);
writer.WriteLine();
writer.WriteLine("var ports = new Dictionary<string, object> { { \"idataset\", idataset } };");
writer.WriteLine("var parameters = new Dictionary<string, object>");
writer.WriteLine("{");
using (writer.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateDictionaryEntry(writer, arg, "");
}
}
writer.WriteLine("};");
writer.WriteLine();
writer.WriteLine("instance.Context = context;");
writer.WriteLine("instance.SetInputPorts(ports);");
writer.WriteLine("instance.SetParameters(parameters);");
writer.WriteLine();
writer.WriteLine("return instance;");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("[Obsolete]");
writer.WriteLine("public class {0}Module : ModuleBase", _compName);
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("private Dictionary<string, object> parameters;");
writer.WriteLine("private Dictionary<string, object> ports;");
writer.WriteLine();
writer.WriteLine("public override Dictionary<string, object> Run()");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var view = ConstructTransform((IDataView)ports[\"idataset\"]);");
writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view }, { \"otransform\", new DataTransform(view) } };");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override void SetParameters(Dictionary<string, object> parameters)");
writer.WriteLine("{");
using (writer.Nest())
writer.WriteLine("this.parameters = parameters;");
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override void SetInputPorts(Dictionary<string, object> ports)");
writer.WriteLine("{");
using (writer.Nest())
writer.WriteLine("this.ports = ports;");
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("public override Dictionary<string, object> ComputeSchema(Dictionary<string, object> inputports)");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var view = ConstructTransform((IDataView)inputports[\"idataset\"]);");
writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view.Schema } };");
}
writer.WriteLine("}");
writer.WriteLine();
writer.WriteLine("private IDataView ConstructTransform(IDataView input)");
writer.WriteLine("{");
using (writer.Nest())
{
writer.WriteLine("var builder = new {0}();", _compName);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplCall(writer, arg, null, null, null, "");
}
writer.WriteLine("return builder.Create{0}Impl(Host, input);", _compName);
}
writer.WriteLine("}");
}
writer.WriteLine("}");
}
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
var className = prefix + component.LoadNames[0];
w.WriteLine("var builder = new {0}();", className);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplCall(w, arg, "");
}
w.WriteLine("var env = new LocalEnvironment(1, verbose: true);");
w.WriteLine("var view = builder.Create{0}{1}Impl(env, data);", prefix, component.LoadNames[0]);
w.WriteLine("return new Tuple<IDataView, DataTransform>(view, new DataTransform(view));");
}
w.WriteLine("}");
}
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("/// <summary>");
w.WriteLine("/// Creates a {0}", component.LoadNames[0]);
w.WriteLine("/// </summary>");
w.WriteLine("/// <param name=\"env\">The environment</param>");
w.WriteLine("/// <param name=\"data\">The data set</param>");
w.WriteLine("/// <returns>The transformed data.</returns>");
w.WriteLine("[Obsolete]");
w.WriteLine("public IDataView Create{0}{1}Impl(", prefix, component.LoadNames[0]);
using (w.Nest())
{
w.WriteLine("IHostEnvironment env,");
w.WriteLine("IDataView data)");
}
}
// This is for "forking" the host environment.
private CrossValidationCommand(CrossValidationCommand impl)
: base(impl, RegistrationName)
{
_info = impl._info;
}
protected abstract void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component);
protected abstract void GenerateMethodSignature(IndentingTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component);
public static bool IsOfType(this ComponentCatalog.LoadableClassInfo component, Type type)
{
return(component.SignatureTypes != null && component.SignatureTypes.Contains(type));
}
protected override void GenerateContent(IndentingTextWriter writer, string prefix, ComponentCatalog.LoadableClassInfo component, string moduleId)
{
GenerateImplFields(writer, component, (w, a) => GenerateFieldsOrProperties(w, a, "", GenerateField));
GenerateImplFields(writer, component, (w, a) => GenerateFieldsOrProperties(w, a, "", GenerateProperty));
GenerateMethodSignature(writer, prefix, component);
GenerateImplBody(writer, component);
}
