public override void Run() { string command = "Test"; using (var ch = Host.Start(command)) using (var server = InitServer(ch)) { var settings = CmdParser.GetSettings(Host, ImplOptions, new Arguments()); ch.Info("maml.exe {0} {1}", command, settings); SendTelemetry(Host); using (new TimerScope(Host, ch)) { RunCore(ch); } } }
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("[Obsolete]"); w.Write("public static Tuple<ITrainer> Create{0}{1}(", prefix, component.LoadNames[0]); using (w.Nest()) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray(); var pre = ""; foreach (var arg in arguments) { GenerateMethodSignature(w, arg, null, null, null, ref pre, ""); } w.WriteLine(")"); } }
public void TestCommandLineParserWithotRequiredArgumentsError()//MissingRequiredOptionError { var args = new[] { "--db-server", "name", "--folder", "c://..." }; var result = CmdParser.Parse(args); Assert.AreEqual(result.IsSuccess, false); var args2 = new[] { "--mode", "import-all", "--folder", "c://..." }; result = CmdParser.Parse(args2); Assert.AreEqual(result.IsSuccess, false); var args3 = new[] { "--mode", "export-list" }; result = CmdParser.Parse(args3); Assert.AreEqual(result.IsSuccess, false); }
protected override void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType)); w.WriteLine("var defs = new {0}();", GetCSharpTypeName(component.ArgType)); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray(); foreach (var arg in arguments) { GenerateImplBody(w, arg, ""); } w.WriteLine("return new Tuple<string, string>(\"{0}\", CmdParser.GetSettings(args, defs));", component.LoadNames[0]); } w.WriteLine("}"); }
public static IDataView GetKcHouseDataView(string dataPath) { var dataSchema = "col=Id:TX:0 col=Date:TX:1 col=Label:R4:2 col=Bedrooms:R4:3 " + "col=Bathrooms:R4:4 col=SqftLiving:R4:5 col=SqftLot:R4:6 col=Floors:R4:7 " + "col=Waterfront:R4:8 col=View:R4:9 col=Condition:R4:10 col=Grade:R4:11 " + "col=SqftAbove:R4:12 col=SqftBasement:R4:13 col=YearBuilt:R4:14 " + "col=YearRenovated:R4:15 col=Zipcode:R4:16 col=Lat:R4:17 col=Long:R4:18 " + "col=SqftLiving15:R4:19 col=SqftLot15:R4:20 header+ sep=,"; var txtArgs = new Runtime.Data.TextLoader.Arguments(); bool parsed = CmdParser.ParseArguments(s_environment, dataSchema, txtArgs); s_environment.Assert(parsed); var txtLoader = new Runtime.Data.TextLoader(s_environment, txtArgs, new MultiFileSource(dataPath)); return(txtLoader); }
public void TestValidCommandLineArguments() { CmdParser cmdParser = this.SpawnParser(); string[] args = new string[] { "/t1=\"in 1\"", "/t2", "/t3=input3" }; bool result = cmdParser.ParseArgs(args); Assert.IsTrue(result, "Valid argument parsing should return true for ParseArgs"); Assert.AreEqual <string>("\"in 1\"", cmdParser.GetArg("t1"), "Correct argument should match by key and be return as quoted if entered as such"); Assert.AreEqual <string>("", cmdParser.GetArg("t2"), "Argument value without input should simply be an empty string"); Assert.AreEqual <string>("input3", cmdParser.GetArg("t3"), "Correct argument should match by key and be returned without quotes if none were provided"); }
public override void Run() { using (var ch = Host.Start(LoadName)) using (var server = InitServer(ch)) { var settings = CmdParser.GetSettings(Host, ImplOptions, new Arguments()); string cmd = string.Format("maml.exe {0} {1}", LoadName, settings); ch.Info(cmd); SendTelemetry(Host); using (new TimerScope(Host, ch)) { RunCore(ch, cmd); } } }
protected override void GenerateMethodSignature(IndentingTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("public static Tuple<IDataView, DataTransform> Create{0}{1}(", prefix, component.LoadNames[0]); using (w.Nest()) { var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); w.WriteLine("[DataLabInputPort(FriendlyName = \"IDataView\", DisplayName = \"IDataView\", IsOptional = false, DataTypes = WellKnownDataTypeIds.IDataViewDotNet, Description = \"Input data (IDataView)\")]"); w.Write("IDataView data"); var pre = ","; foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateMethodSignature(w, arg, null, null, null, ref pre, ""); } w.WriteLine(")"); } }
protected override void GenerateImplCall(IndentingTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { var className = prefix + component.LoadNames[0]; w.WriteLine("var builder = new {0}();", className); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray(); foreach (var arg in arguments) { GenerateImplCall(w, arg, ""); } w.WriteLine("var learner = builder.GetTlcSettings();"); w.WriteLine("return new TlcTrainer(learner.Item1, learner.Item2);"); } w.WriteLine("}"); }
public override string ToString() { if (IsEmpty) { return("{}"); } if (_settings.Length == 0) { return(_kind); } string str = CmdParser.CombineSettings(_settings); StringBuilder sb = new StringBuilder(); CmdQuoter.QuoteValue(str, sb, true); return(_kind + sb.ToString()); }
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("{"); using (w.Nest()) { var className = prefix + component.LoadNames[0]; w.WriteLine("var builder = new {0}();", className); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateImplCall(w, arg, ""); } w.WriteLine("var env = new LocalEnvironment(1, verbose: true);"); w.WriteLine("var view = builder.Create{0}{1}Impl(env, data);", prefix, component.LoadNames[0]); w.WriteLine("return new Tuple<IDataView, DataTransform>(view, new DataTransform(view));"); } w.WriteLine("}"); }
public void TestPcaTransform() { // Force Microsoft.ML.Runtime.PCA assembly to be loaded into the AppDomain so // ReflectionUtils.FindClassCore does not return null when called by ReflectionUtils.CreateInstance Assert.AreEqual(typeof(PCAPredictor).Name, "PCAPredictor"); string trainData = GetDataPath(TestDatasets.mnistTiny28.trainFilename); string fileName = TestContext.TestName + "-Train.txt"; string outFile = DeleteOutputPath(Dir, fileName); const int rank = 3; string pcaTransformArgs = string.Format("/inst Trans{{trans=pca {{k={0} seed=1}}}}", rank); var argsStr1 = string.Format( "/c CreateInstances {0} {1} /rs=1 /cifile={2}", trainData, pcaTransformArgs, outFile); var args1 = new TLCArguments(); Assert.IsTrue(CmdParser.ParseArguments(argsStr1, args1)); RunExperiments.Run(args1); CheckEquality(Dir, fileName); // Verify the scales of the transformed features decrease with respect to the feature index TlcTextInstances outputInstances = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile); Double[] l1norms = new Double[rank]; foreach (Instance instance in outputInstances) { Assert.IsTrue(instance.Features.Count == rank); for (int i = 0; i < instance.Features.Values.Length; i++) { l1norms[i] += (instance.Features.Values[i] < 0 ? -instance.Features.Values[i] : instance.Features.Values[i]); } } for (int i = 0; i < l1norms.Length - 1; i++) { Assert.IsTrue(l1norms[i] > l1norms[i + 1]); } Done(); }
public void TestCreateTextInstancesWithNormalization() { TestDataset dataset = TestDatasets.mnistTiny28; string trainData = GetDataPath(dataset.trainFilename); string testData = GetDataPath(dataset.testFilename); var prefix = TestContext.TestName + "-"; string outFile1 = DeleteOutputPath(Dir, prefix + "Norm-Separate-Train.txt"); string outTestFile1 = DeleteOutputPath(Dir, prefix + "Norm-Separate-Test.txt"); string outFile2 = DeleteOutputPath(Dir, prefix + "Norm-Trans-Train.txt"); string outTestFile2 = DeleteOutputPath(Dir, prefix + "Norm-Trans-Test.txt"); string transArgs = "inst=Trans{trans=RFF {rng=1}}"; var argsStr1 = string.Format( "/c=CreateInstances {0} /test={1} /norm=MinMaxNormalizer /{2} /cifile={3} /citestfile={4}", trainData, testData, transArgs, outFile1, outTestFile1); var args1 = new TLCArguments(); Assert.IsTrue(CmdParser.ParseArguments(argsStr1, args1)); RunExperiments.Run(args1); var argsStr2 = string.Format( "/c=CreateInstances {0} /test={1} /inst Trans{{trans=MinMaxNormalizer {2}}} /cifile={3} /citestfile={4}", trainData, testData, transArgs, outFile2, outTestFile2); var args2 = new TLCArguments(); Assert.IsTrue(CmdParser.ParseArguments(argsStr2, args2)); RunExperiments.Run(args2); var instances1 = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile1); var instances2 = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile2); CompareInstances(instances1, instances2); var testInstances1 = new TlcTextInstances(new TlcTextInstances.Arguments(), outTestFile1); var testInstances2 = new TlcTextInstances(new TlcTextInstances.Arguments(), outTestFile2); CompareInstances(testInstances1, testInstances2); Done(); }
public override bool Parse(string cmd) { var parser = new CmdParser(cmd); switch (parser.Command) { case "exit": fileManager?.Dispose(); return(false); case "gc": GC.Collect(); break; case "close": fileManager?.Dispose(); break; case "make": MakeDatabase(parser); break; case "open": LoadDatabase(parser); break; case "adddata": AddData(parser); break; case "getdata": GetData(parser); break; case "vacuum": fileManager.DataVacuum(); break; case "trace": Trace(parser); break; } return(true); }
public void TestCreateTextInstances() { TestDataset adult = TestDatasets.adult; string trainData = GetDataPath(adult.trainFilename); string testData = GetDataPath(adult.testFilename); var prefix = TestContext.TestName + "-"; string outName = prefix + "Adult-Train.txt"; string statsName = prefix + "Adult-Train.stats.txt"; string outTestName = prefix + "Adult-Test.txt"; string testStatsName = prefix + "Adult-Test.stats.txt"; string outValidName = prefix + "Adult-Valid.txt"; string validStatsName = prefix + "Adult-Valid.stats.txt"; string outFile = DeleteOutputPath(Dir, outName); string statsFile = DeleteOutputPath(Dir, statsName); string outTestFile = DeleteOutputPath(Dir, outTestName); string testStatsFile = DeleteOutputPath(Dir, testStatsName); string outValidFile = DeleteOutputPath(Dir, outValidName); string validStatsFile = DeleteOutputPath(Dir, validStatsName); var argsStr = string.Format( "/c=CreateInstances {0} /test={1} /valid={1} /cacheinst=- {2} " + "/cifile={3} /cistatsfile={4} /citestfile={5} /citeststatsfile={6} /civalidfile={7} /civalidstatsfile={8}", trainData, testData, adult.extraSettings, outFile, statsFile, outTestFile, testStatsFile, outValidFile, validStatsFile); argsStr += " /writer TextInstanceWriter{/stats=+} /disableTracking=+"; var args = new TLCArguments(); Assert.IsTrue(CmdParser.ParseArguments(argsStr, args)); RunExperiments.Run(args); CheckEquality(Dir, outName); CheckEquality(Dir, statsName); CheckEquality(Dir, outTestName); CheckEquality(Dir, testStatsName); CheckEquality(Dir, outValidName); CheckEquality(Dir, validStatsName); Done(); }
public void Trace(CmdParser parser) { var type = parser.GetAttribute("type") ?? parser.GetAttribute(0); switch (type) { case "d": Console.WriteLine(fileManager.TraceDirs()); break; case "f": Console.WriteLine(fileManager.TraceFiles()); break; default: Console.WriteLine(fileManager); break; } }
public void WriteTo(CmdParser parser) { var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0); var outFilePath = parser.GetAttribute("out") ?? parser.GetAttribute(1); //var parent = parser.GetAttribute("parent") ?? parser.GetAttribute(2) ?? "/"; if (fullPath.Substring(0, 1).Equals(":")) { var id = fullPath.TrimStart(':').ToInt(); //fileManager.WriteToFile(id, outFilePath); fileManager.WriteToFile(id, outFilePath); } else { fileManager.WriteToFile(fullPath, outFilePath); //fileManager.WriteToFile(fullPath, outFilePath); } }
protected void GenerateSummaryComment(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix) { if (Exclude.Contains(arg.LongName)) { return; } if (arg.IsSubComponentItemType) { // We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer. Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>)); var types = arg.ItemType.GetGenericArguments(); var baseType = types[0]; var sigType = types[1]; if (sigType == typeof(SignatureDataLoader)) { return; } GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText); if (IsTrainer(sigType)) { return; } var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType); foreach (var info in infos) { var args = info.CreateArguments(); if (args == null) { continue; } var argInfo = CmdParser.GetArgInfo(args.GetType(), args); foreach (var a in argInfo.Args) { GenerateSummaryComment(w, a, argSuffix + info.LoadNames[0]); } } } else { GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText); } }
private static TRes CreateCore <TRes>( IHostEnvironment env, Type factoryType, Type signatureType, string settings, out string loadName, params object[] extraArgs) where TRes : class { Contracts.AssertValue(env); env.AssertValue(factoryType); env.AssertValue(signatureType); env.AssertValue(settings, "settings"); var factory = CmdParser.CreateComponentFactory(factoryType, signatureType, settings); loadName = factory.Name; return(ComponentCatalog.CreateInstance <TRes>(env, factory.SignatureType, factory.Name, factory.GetSettingsString(), extraArgs)); }
public override void Run() { string command = "Train"; using (var ch = Host.Start(command)) using (var server = InitServer(ch)) { var settings = CmdParser.GetSettings(Host, Args, new Arguments()); string cmd = string.Format("maml.exe {0} {1}", command, settings); ch.Info(cmd); SendTelemetry(Host); using (new TimerScope(Host, ch)) { RunCore(ch, cmd); } } }
public void TestInvalidCommandLineArguments() { using (StringWriter sw = new StringWriter()) { Console.SetOut(sw); CmdParser cmdParser = this.SpawnParser(); string[] args = new string[] { "t1=\"in 1\"", "/t2", "/t3=\"in 3\"" }; bool result = cmdParser.ParseArgs(args); Assert.IsFalse(result, "Invalid argument should return false on ParseArgs"); cmdParser.PrintErrors(); string expected = string.Format(" Invalid command line argument: t1=\"in 1\"{0}", Environment.NewLine); Assert.AreEqual <string>(expected, sw.ToString(), " Invalid command line argument: t1=\"in 1\""); } }
static void Main(string[] args) { try { var arguments = CmdParser.Parse(args); var output = InputParser.Parse(arguments.FilePath); output = FilterHandler.Filter(output, arguments); OutputPrinter.Print(output); Console.ReadKey(); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.ReadKey(); } }
public void GetFiles(CmdParser parser) { var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0); FileStructure[] fileStructures; if (fullPath.Substring(0, 1).Equals(":")) { var id = fullPath.TrimStart(':').ToInt(); fileStructures = fileManager.GetFiles(id); } else { fileStructures = fileManager.GetFiles(fullPath); } foreach (var file in fileStructures) { Console.WriteLine("{0}", file); } }
public void GetDirs(CmdParser parser) { var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0); DirectoryStructure[] directoryStructures; if (fullPath.Substring(0, 1).Equals(":")) { var id = fullPath.TrimStart(':').ToInt(); directoryStructures = fileManager.GetDirectories(id); } else { directoryStructures = fileManager.GetDirectories(fullPath); } foreach (var dir in directoryStructures) { Console.WriteLine("{0}", dir); } }
public void TestMissingCommandLineArguments() { using (StringWriter sw = new StringWriter()) { Console.SetOut(sw); CmdParser cmdParser = this.SpawnParser(); string[] args = new string[] { "/t1=\"in 1\"", "/t2", }; bool result = cmdParser.ParseArgs(args); Assert.IsFalse(result, "Missing argument should return false on ParseArgs"); cmdParser.PrintErrors(); string expected = string.Format(" Missing command line argument: t3{0}", Environment.NewLine); Assert.AreEqual <string>(expected, sw.ToString(), "Missing arguments should provide accurate message"); } }
public void NewTest() { var ml = new MLContext(); ml.AddStandardComponents(); var classes = Utils.MarshalInvoke(ml.ComponentCatalog.FindLoadableClasses <int>, typeof(SignatureCalibrator)); foreach (var cls in classes) { var factory = CmdParser.CreateComponentFactory(typeof(IComponentFactory <ICalibratorTrainer>), typeof(SignatureCalibrator), cls.LoadNames[0]); var calibrator = ((IComponentFactory <ICalibratorTrainer>)factory).CreateComponent(ml); } var components = ml.ComponentCatalog.GetAllComponents(typeof(ICalibratorTrainerFactory)); foreach (var component in components) { var factory = CmdParser.CreateComponentFactory(typeof(IComponentFactory <ICalibratorTrainer>), typeof(SignatureCalibrator), component.Aliases[0]); var calibrator = ((IComponentFactory <ICalibratorTrainer>)factory).CreateComponent(ml); } }
/// <summary> /// Creates a data scorer from the 'LoadName{settings}' string. /// </summary> /// <param name="env">The host environment to use.</param> /// <param name="settings">The settings string.</param> /// <param name="data">The data to score.</param> /// <param name="predictor">The predictor to score.</param> /// <param name="trainSchema">The training data schema from which the scorer can optionally extract /// additional information, for example, label names. If this is <c>null</c>, no information will be /// extracted.</param> /// <returns>The scored data.</returns> internal static IDataScorerTransform CreateScorer(this IHostEnvironment env, string settings, RoleMappedData data, IPredictor predictor, RoleMappedSchema trainSchema = null) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(data, nameof(data)); env.CheckValue(predictor, nameof(predictor)); env.CheckValueOrNull(trainSchema); Type factoryType = typeof(IComponentFactory <IDataView, ISchemaBoundMapper, RoleMappedSchema, IDataScorerTransform>); Type signatureType = typeof(SignatureDataScorer); ICommandLineComponentFactory scorerFactorySettings = CmdParser.CreateComponentFactory( factoryType, signatureType, settings); var bindable = ScoreUtils.GetSchemaBindableMapper(env, predictor, scorerFactorySettings: scorerFactorySettings); var mapper = bindable.Bind(env, data.Schema); return(CreateCore <IDataScorerTransform>(env, factoryType, signatureType, settings, data.Data, mapper, trainSchema)); }
// [TestCase("blablabla")] public void UpdateNonExistingTableTest(string nonExistingTableId) { string updateFileName = "filetoupdate" + DateTime.Now.Ticks + ".csv"; string[] args = new string[] { "write-table", nonExistingTableId, updateFileName, "-t", testToken }; //creates test file FileInfo updateFileInfo = new FileInfo(updateFileName); string[] dataToUpdate = PrepareDataToUpdate(100); System.IO.File.WriteAllLines(updateFileInfo.FullName, dataToUpdate); var command = CmdParser.ParseArguments(args); Controller ctrl = new Controller(command); bool result = ctrl.Execute(); Assert.That(result, Is.False); updateFileInfo.Delete(); }
public void TestCommandLineParserUsingMaxVersion() { var args = new[] { "--mode", "export-list", "--db-server", "name", "--rule-names", "rName1" }; var result = CmdParser.Parse(args); var same = new Args { Mode = Mode.ExportList, BreRecords = new List <BreRecord>() }; var rule = new BreRecord() { UseMaxVersion = true }; same.RuleNames.Add(rule); Assert.AreEqual(result.Mode, same.Mode); for (var i = 0; i < result.RuleNames.Count; i++) { Assert.AreEqual(result.RuleNames[i].UseMaxVersion, same.RuleNames[i].UseMaxVersion); } }
protected override void GenerateSummaryComment(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component) { w.WriteLine("/// <summary>"); var desc = component.Summary ?? component.LoadNames[0]; using (var sr = new StringReader(desc)) { string line; while ((line = sr.ReadLine()) != null) { w.WriteLine("/// {0}", line); } } w.WriteLine("/// </summary>"); GenerateParameterComment(w, "data", "The data"); var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments()); foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden)) { GenerateSummaryComment(w, arg, ""); } }