protected void GenerateModuleType(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component) { string cat; if (component.IsOfType(typeof(SignatureBinaryClassifierTrainer))) { cat = "BinaryClassifier"; } else if (component.IsOfType(typeof(SignatureMultiClassClassifierTrainer))) { cat = "MultiClassClassifier"; } else if (component.IsOfType(typeof(SignatureRegressorTrainer))) { cat = "Regression"; } else if (component.IsOfType(typeof(SignatureAnomalyDetectorTrainer))) { cat = "AnomalyDetector"; } else { cat = "None"; } w.WriteLine("[DataLabModuleType(Type = ModuleType.{0})]", cat); }
protected override void GenerateEnumValue(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo info) { var name = info != null ? info.LoadNames[0] : "None"; w.WriteLine("/// <summary> {0} option </summary>", name); w.Write("{0}", name); }
public CrossValidationCommand(IHostEnvironment env, Arguments args) : base(env, args, RegistrationName) { Host.CheckUserArg(Args.NumFolds >= 2, nameof(Args.NumFolds), "Number of folds must be greater than or equal to 2."); _info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile); Utils.CheckOptionalUserDirectory(Args.SummaryFilename, nameof(Args.SummaryFilename)); Utils.CheckOptionalUserDirectory(Args.OutputDataFile, nameof(Args.OutputDataFile)); }
protected override void GenerateEnumValue(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo info) { var userName = info != null ? info.UserName : "******"; var name = info != null ? info.LoadNames[0] : "None"; w.WriteLine("[ItemInfo(FriendlyName = \"{0}\", DisplayValue = \"{1}\")]", userName, userName); w.Write("{0}", name); }
/// <summary> /// Determine the default scorer for a schema bound mapper. This looks for text-valued ScoreColumnKind /// metadata on the first column of the mapper. If that text is found and maps to a scorer loadable class, /// that component is used. Otherwise, the GenericScorer is used. /// </summary> /// <param name="environment">The host environment.</param>. /// <param name="mapper">The schema bound mapper to get the default scorer.</param>. /// <param name="suffix">An optional suffix to append to the default column names.</param> public static TScorerFactory GetScorerComponent( IHostEnvironment environment, ISchemaBoundMapper mapper, string suffix = null) { Contracts.CheckValue(environment, nameof(environment)); Contracts.AssertValue(mapper); ComponentCatalog.LoadableClassInfo info = null; ReadOnlyMemory <char> scoreKind = default; if (mapper.OutputSchema.Count > 0 && mapper.OutputSchema.TryGetMetadata(TextType.Instance, MetadataUtils.Kinds.ScoreColumnKind, 0, ref scoreKind) && !scoreKind.IsEmpty) { var loadName = scoreKind.ToString(); info = environment.ComponentCatalog.GetLoadableClassInfo <SignatureDataScorer>(loadName); if (info == null || !typeof(IDataScorerTransform).IsAssignableFrom(info.Type)) { info = null; } } Func <IHostEnvironment, IDataView, ISchemaBoundMapper, RoleMappedSchema, IDataScorerTransform> factoryFunc; if (info == null) { factoryFunc = (env, data, innerMapper, trainSchema) => new GenericScorer( env, new GenericScorer.Arguments() { Suffix = suffix }, data, innerMapper, trainSchema); } else { factoryFunc = (env, data, innerMapper, trainSchema) => { object args = info.CreateArguments(); if (args is ScorerArgumentsBase scorerArgs) { scorerArgs.Suffix = suffix; } return((IDataScorerTransform)info.CreateInstance( env, args, new object[] { data, innerMapper, trainSchema })); }; } return(ComponentFactoryUtils.CreateFromFunction(factoryFunc)); }
protected void GenerateEnums(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var seen = new HashSet <Tuple <Type, Type> >(); foreach (var arg in argumentInfo.Args) { GenerateEnums(w, arg, seen); } }
public Component(string kind, ComponentCatalog.LoadableClassInfo info, object args) { Contracts.AssertNonEmpty(kind); Contracts.AssertValue(info); Contracts.AssertValueOrNull(args); Kind = kind; Info = info; Args = args; }
public TrainTestCommand(IHostEnvironment env, Arguments args) : base(env, args, nameof(TrainTestCommand)) { Utils.CheckOptionalUserDirectory(args.SummaryFilename, nameof(args.SummaryFilename)); Utils.CheckOptionalUserDirectory(args.OutputDataFile, nameof(args.OutputDataFile)); _info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile); if (string.IsNullOrWhiteSpace(args.TestFile)) { throw Host.ExceptUserArg(nameof(args.TestFile), "Test file must be defined."); } }
protected void GenerateImplFields(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component, Action <IndentingTextWriter, CmdParser.ArgInfo.Arg> fieldGenerator) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray(); foreach (var arg in arguments) { fieldGenerator(w, arg); } }
public TrainCommand(IHostEnvironment env, Arguments args) : base(env, args, nameof(TrainCommand)) { Host.CheckNonWhiteSpace(args.OutputModelFile, nameof(args.OutputModelFile)); _info = TrainUtils.CheckTrainer(Host, args.Trainer, args.DataFile); _trainer = args.Trainer; _labelColumn = args.LabelColumn; _featureColumn = args.FeatureColumn; _groupColumn = args.GroupColumn; _weightColumn = args.WeightColumn; _nameColumn = args.NameColumn; }
private void GenerateFile(ComponentCatalog.LoadableClassInfo info, string filename, Dictionary <Type, GeneratorBase> mapping, bool generateEnums) { using (var sw = new StreamWriter(filename)) { var writer = IndentingTextWriter.Wrap(sw, " "); foreach (var kvp in mapping) { if (info.IsOfType(kvp.Key)) { kvp.Value.Generate(writer, _modulePrefix, _regenerate, info, generateEnums, _moduleId ?? Guid.NewGuid().ToString(), _moduleName, _moduleOwner, _moduleVersion, _moduleState, _moduleType, _moduleDeterminism, _moduleCategory, _exclude, _namespaces); break; } } } }
protected override void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType)); w.WriteLine("var defs = new {0}();", GetCSharpTypeName(component.ArgType)); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray(); foreach (var arg in arguments) { GenerateImplBody(w, arg, ""); } w.WriteLine("return new Tuple<string, string>(\"{0}\", CmdParser.GetSettings(args, defs));", component.LoadNames[0]); } w.WriteLine("}"); }
protected override void GenerateMethodSignature(IndentingTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("public static Tuple<IDataView, DataTransform> Create{0}{1}(", prefix, component.LoadNames[0]); using (w.Nest()) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); w.WriteLine("[DataLabInputPort(FriendlyName = \"IDataView\", DisplayName = \"IDataView\", IsOptional = false, DataTypes = WellKnownDataTypeIds.IDataViewDotNet, Description = \"Input data (IDataView)\")]"); w.Write("IDataView data"); var pre = ","; foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateMethodSignature(w, arg, null, null, null, ref pre, ""); } w.WriteLine(")"); } }
private string GenerateCall(ComponentCatalog.LoadableClassInfo component) { // The caller needs to ensure that the component has either a constructor or a create method. Contracts.Assert(component.Constructor != null || component.CreateMethod != null); string call; if (component.Constructor != null) { var type = GetCSharpTypeName(component.Constructor.DeclaringType); call = string.Format("new {0}", type); } else { var type = GetCSharpTypeName(component.CreateMethod.DeclaringType); var name = component.CreateMethod.Name; call = string.Format("{0}.{1}", type, name); } return(call); }
protected override void GenerateModuleAttribute(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component, string moduleId) { if (!string.IsNullOrEmpty(prefix)) { prefix += " "; } w.WriteLine("[DataLabModule(FriendlyName = \"{0}{1}\",", prefix, component.UserName); using (w.Nest()) { var desc = component.Summary ?? component.LoadNames[0]; w.WriteLine("Description = \"{0}\",", desc.Replace("\n", "\\n").Replace("\r", "\\r")); w.WriteLine("IsBlocking = true,"); w.WriteLine("IsDeterministic = true,"); w.WriteLine("Version = \"2.0\","); w.WriteLine("Owner = \"Microsoft Corporation\","); w.WriteLine("FamilyId = \"{{{0}}}\",", moduleId.ToUpperInvariant()); w.WriteLine("ReleaseState = States.Alpha)]"); } }
protected override void GenerateSummaryComment(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("/// <summary>"); var desc = component.Summary ?? component.LoadNames[0]; using (var sr = new StringReader(desc)) { string line; while ((line = sr.ReadLine()) != null) { w.WriteLine("/// {0}", line); } } w.WriteLine("/// </summary>"); GenerateParameterComment(w, "data", "The data"); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateSummaryComment(w, arg, ""); } }
private void GenerateFile(IChannel ch, ComponentCatalog.LoadableClassInfo info) { _host.AssertValue(ch); ch.AssertValue(info); string name = info.LoadNames[0]; if (!info.IsOfType(typeof(SignatureTrainer)) && !info.IsOfType(typeof(SignatureDataTransform))) { ch.Warning("No generator available for {0}.", name); return; } if (info.Constructor == null && info.CreateMethod == null) { ch.Warning("No construction method available for {0}.", name); return; } if (_generateModule) { var entryPointFile = _modulePrefix + name + "EntryPoint.cs"; if (_generateModuleInstance) { GenerateFile(info, entryPointFile, ModuleInstanceEntryPointGeneratorMapping); } else { GenerateFile(info, entryPointFile, EntryPointGeneratorMapping); } } var implFile = _modulePrefix + name + ".cs"; GenerateFile(info, implFile, ImplGeneratorMapping); }
protected override void GenerateImplBody(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { if (component.ArgType == null) { var call = GenerateCall(component); w.WriteLine("return {0}(env, data);", call); } else { w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType)); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateImplBody(w, arg, ""); } var call = GenerateCall(component); w.WriteLine("return {0}(args, env, data);", call); } } w.WriteLine("}"); }
public static List <string> GetLearnerSettingsAndSweepParams(IHostEnvironment env, ComponentCatalog.LoadableClassInfo cl, out string settings) { List <string> sweepParams = new List <string>(); var ci = cl.Constructor?.GetParameters(); if (ci == null) { settings = ""; return(sweepParams); } var suggestedSweepsParser = new SuggestedSweepsParser(); StringBuilder learnerSettings = new StringBuilder(); foreach (var prop in ci) { var fieldInfo = prop.ParameterType?.GetFields(BindingFlags.Public | BindingFlags.Instance); foreach (var field in fieldInfo) { TGUIAttribute[] tgui = field.GetCustomAttributes(typeof(TGUIAttribute), true) as TGUIAttribute[]; if (tgui == null) { continue; } foreach (var attr in tgui) { if (attr.SuggestedSweeps != null) { // Build the learner setting. learnerSettings.Append($" {field.Name}=${field.Name}$"); // Build the sweeper. suggestedSweepsParser.TryParseParameter(attr.SuggestedSweeps, field.FieldType, field.Name, out var sweepValues, out var error); sweepParams.Add(sweepValues?.ToStringParameter(env)); } } } } settings = learnerSettings.ToString(); return(sweepParams); }
/// <summary> /// Generate the module and its implementation. /// </summary> /// <param name="writer">The writer.</param> /// <param name="prefix">The module prefix.</param> /// <param name="regenerate">The command string used to generate.</param> /// <param name="component">The component.</param> /// <param name="moduleId"></param> /// <param name="moduleName"></param> /// <param name="moduleOwner"></param> /// <param name="moduleVersion"></param> /// <param name="moduleState"></param> /// <param name="moduleType"></param> /// <param name="moduleDeterminism"></param> /// <param name="moduleCategory"></param> /// <param name="exclude">The set of parameters to exclude</param> /// <param name="namespaces">The set of extra namespaces</param> public void Generate(IndentedTextWriter writer, string prefix, string regenerate, ComponentCatalog.LoadableClassInfo component, string moduleId, string moduleName, string moduleOwner, string moduleVersion, string moduleState, string moduleType, string moduleDeterminism, string moduleCategory, HashSet <string> exclude, HashSet <string> namespaces) { Contracts.AssertValue(exclude); Name = moduleName; Owner = moduleOwner; Version = moduleVersion; State = moduleState; ModuleType = moduleType; Determinism = moduleDeterminism; Category = moduleCategory; Exclude = exclude; Namespaces = namespaces; GenerateHeader(writer, regenerate); using (writer.Nest()) { GenerateClassName(writer, prefix, component); using (writer.Nest()) GenerateContent(writer, prefix, component, moduleId); GenerateFooter(writer); } GenerateFooter(writer); }
protected virtual void GenerateClassName(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine(); var className = prefix + component.LoadNames[0]; w.WriteLine("/// <summary>Module: {0}</summary>", className); w.WriteLine("public partial class {0}", className); w.WriteLine("{"); }
protected override void GenerateClassName(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine(); var className = prefix + component.LoadNames[0]; w.WriteLine("/// <summary>Module: {0}</summary>", className); w.WriteLine("[Obsolete]"); w.WriteLine("public static class {0}EntryPoint", className); w.WriteLine("{"); }
protected override void GenerateContent(IndentedTextWriter writer, string prefix, ComponentCatalog.LoadableClassInfo component, string moduleId) { writer.WriteLine("[Module("); _compName = prefix + component.LoadNames[0]; var name = Name ?? PrettyPrintDisplayName(component.LoadNames[0]); using (writer.Nest()) { writer.WriteLine("Name = \"{0}\",", name); writer.WriteLine("FamilyId = \"{0}\",", moduleId); writer.WriteLine("Owner = \"{0}\",", Owner); writer.WriteLine("ReleaseVersion = \"{0}\",", Version); writer.WriteLine("State = ModuleState.{0},", State); writer.WriteLine("Type = ModuleType.{0},", ModuleType); writer.WriteLine("Determinism = Determinism.{0},", Determinism); writer.WriteLine("Category = @\"{0}\")]", Category); } writer.WriteLine("[Obsolete]"); writer.WriteLine("public static IModule Create{0}(", _compName); using (writer.Nest()) { writer.WriteLine("[Help(Display = @\"Dataset\", ToolTip = @\"Input dataset\")]"); writer.WriteLine("[ModuleInputPort]"); writer.WriteLine("IDataView idataset,"); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateMethodSignature(writer, arg, null, null, null, ""); } writer.WriteLine("[Help(Display = @\"Results dataset\", ToolTip = @\"Transformed dataset\")]"); writer.WriteLine("[ModuleOutputPort]"); writer.WriteLine("IDataView odataset,"); writer.WriteLine("[Help(Display = @\"{0}\", ToolTip = @\"{0}\")]", name); writer.WriteLine("[ModuleOutputPort]"); writer.WriteLine("DataTransform otransform,"); writer.WriteLine("[Context]"); writer.WriteLine("IContext context)"); } writer.WriteLine("{"); using (writer.Nest()) { writer.WriteLine("var instance = new {0}Module();", _compName); writer.WriteLine(); writer.WriteLine("var ports = new Dictionary<string, object> { { \"idataset\", idataset } };"); writer.WriteLine("var parameters = new Dictionary<string, object>"); writer.WriteLine("{"); using (writer.Nest()) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateDictionaryEntry(writer, arg, ""); } } writer.WriteLine("};"); writer.WriteLine(); writer.WriteLine("instance.Context = context;"); writer.WriteLine("instance.SetInputPorts(ports);"); writer.WriteLine("instance.SetParameters(parameters);"); writer.WriteLine(); writer.WriteLine("return instance;"); } writer.WriteLine("}"); writer.WriteLine(); writer.WriteLine("[Obsolete]"); writer.WriteLine("public class {0}Module : ModuleBase", _compName); writer.WriteLine("{"); using (writer.Nest()) { writer.WriteLine("private Dictionary<string, object> parameters;"); writer.WriteLine("private Dictionary<string, object> ports;"); writer.WriteLine(); writer.WriteLine("public override Dictionary<string, object> Run()"); writer.WriteLine("{"); using (writer.Nest()) { writer.WriteLine("var view = ConstructTransform((IDataView)ports[\"idataset\"]);"); writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view }, { \"otransform\", new DataTransform(view) } };"); } writer.WriteLine("}"); writer.WriteLine(); writer.WriteLine("public override void SetParameters(Dictionary<string, object> parameters)"); writer.WriteLine("{"); using (writer.Nest()) writer.WriteLine("this.parameters = parameters;"); writer.WriteLine("}"); writer.WriteLine(); writer.WriteLine("public override void SetInputPorts(Dictionary<string, object> ports)"); writer.WriteLine("{"); using (writer.Nest()) writer.WriteLine("this.ports = ports;"); writer.WriteLine("}"); writer.WriteLine(); writer.WriteLine("public override Dictionary<string, object> ComputeSchema(Dictionary<string, object> inputports)"); writer.WriteLine("{"); using (writer.Nest()) { writer.WriteLine("var view = ConstructTransform((IDataView)inputports[\"idataset\"]);"); writer.WriteLine("return new Dictionary<string, object> { { \"odataset\", view.Schema } };"); } writer.WriteLine("}"); writer.WriteLine(); writer.WriteLine("private IDataView ConstructTransform(IDataView input)"); writer.WriteLine("{"); using (writer.Nest()) { writer.WriteLine("var builder = new {0}();", _compName); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateImplCall(writer, arg, null, null, null, ""); } writer.WriteLine("return builder.Create{0}Impl(Host, input);", _compName); } writer.WriteLine("}"); } writer.WriteLine("}"); }
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { var className = prefix + component.LoadNames[0]; w.WriteLine("var builder = new {0}();", className); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateImplCall(w, arg, ""); } w.WriteLine("var env = new LocalEnvironment(1, verbose: true);"); w.WriteLine("var view = builder.Create{0}{1}Impl(env, data);", prefix, component.LoadNames[0]); w.WriteLine("return new Tuple<IDataView, DataTransform>(view, new DataTransform(view));"); } w.WriteLine("}"); }
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("/// <summary>"); w.WriteLine("/// Creates a {0}", component.LoadNames[0]); w.WriteLine("/// </summary>"); w.WriteLine("/// <param name=\"env\">The environment</param>"); w.WriteLine("/// <param name=\"data\">The data set</param>"); w.WriteLine("/// <returns>The transformed data.</returns>"); w.WriteLine("[Obsolete]"); w.WriteLine("public IDataView Create{0}{1}Impl(", prefix, component.LoadNames[0]); using (w.Nest()) { w.WriteLine("IHostEnvironment env,"); w.WriteLine("IDataView data)"); } }
// This is for "forking" the host environment. private CrossValidationCommand(CrossValidationCommand impl) : base(impl, RegistrationName) { _info = impl._info; }
protected abstract void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component);
protected abstract void GenerateMethodSignature(IndentingTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component);
public static bool IsOfType(this ComponentCatalog.LoadableClassInfo component, Type type) { return(component.SignatureTypes != null && component.SignatureTypes.Contains(type)); }
protected override void GenerateContent(IndentingTextWriter writer, string prefix, ComponentCatalog.LoadableClassInfo component, string moduleId) { GenerateImplFields(writer, component, (w, a) => GenerateFieldsOrProperties(w, a, "", GenerateField)); GenerateImplFields(writer, component, (w, a) => GenerateFieldsOrProperties(w, a, "", GenerateProperty)); GenerateMethodSignature(writer, prefix, component); GenerateImplBody(writer, component); }