public override void Run()
{
string command = "Test";
using (var ch = Host.Start(command))
using (var server = InitServer(ch))
{
var settings = CmdParser.GetSettings(Host, ImplOptions, new Arguments());
ch.Info("maml.exe {0} {1}", command, settings);
SendTelemetry(Host);
using (new TimerScope(Host, ch))
{
RunCore(ch);
}
}
}
protected override void GenerateMethodSignature(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("[Obsolete]");
w.Write("public static Tuple<ITrainer> Create{0}{1}(", prefix, component.LoadNames[0]);
using (w.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
var pre = "";
foreach (var arg in arguments)
{
GenerateMethodSignature(w, arg, null, null, null, ref pre, "");
}
w.WriteLine(")");
}
}
public void TestCommandLineParserWithotRequiredArgumentsError()//MissingRequiredOptionError
{
var args = new[] { "--db-server", "name", "--folder", "c://..." };
var result = CmdParser.Parse(args);
Assert.AreEqual(result.IsSuccess, false);
var args2 = new[] { "--mode", "import-all", "--folder", "c://..." };
result = CmdParser.Parse(args2);
Assert.AreEqual(result.IsSuccess, false);
var args3 = new[] { "--mode", "export-list" };
result = CmdParser.Parse(args3);
Assert.AreEqual(result.IsSuccess, false);
}
protected override void GenerateImplBody(IndentingTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
w.WriteLine("var args = new {0}();", GetCSharpTypeName(component.ArgType));
w.WriteLine("var defs = new {0}();", GetCSharpTypeName(component.ArgType));
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
GenerateImplBody(w, arg, "");
}
w.WriteLine("return new Tuple<string, string>(\"{0}\", CmdParser.GetSettings(args, defs));", component.LoadNames[0]);
}
w.WriteLine("}");
}
public static IDataView GetKcHouseDataView(string dataPath)
{
var dataSchema = "col=Id:TX:0 col=Date:TX:1 col=Label:R4:2 col=Bedrooms:R4:3 " +
"col=Bathrooms:R4:4 col=SqftLiving:R4:5 col=SqftLot:R4:6 col=Floors:R4:7 " +
"col=Waterfront:R4:8 col=View:R4:9 col=Condition:R4:10 col=Grade:R4:11 " +
"col=SqftAbove:R4:12 col=SqftBasement:R4:13 col=YearBuilt:R4:14 " +
"col=YearRenovated:R4:15 col=Zipcode:R4:16 col=Lat:R4:17 col=Long:R4:18 " +
"col=SqftLiving15:R4:19 col=SqftLot15:R4:20 header+ sep=,";
var txtArgs = new Runtime.Data.TextLoader.Arguments();
bool parsed = CmdParser.ParseArguments(s_environment, dataSchema, txtArgs);
s_environment.Assert(parsed);
var txtLoader = new Runtime.Data.TextLoader(s_environment, txtArgs, new MultiFileSource(dataPath));
return(txtLoader);
}
public void TestValidCommandLineArguments()
{
CmdParser cmdParser = this.SpawnParser();
string[] args = new string[]
{
"/t1=\"in 1\"", "/t2", "/t3=input3"
};
bool result = cmdParser.ParseArgs(args);
Assert.IsTrue(result, "Valid argument parsing should return true for ParseArgs");
Assert.AreEqual <string>("\"in 1\"", cmdParser.GetArg("t1"), "Correct argument should match by key and be return as quoted if entered as such");
Assert.AreEqual <string>("", cmdParser.GetArg("t2"), "Argument value without input should simply be an empty string");
Assert.AreEqual <string>("input3", cmdParser.GetArg("t3"), "Correct argument should match by key and be returned without quotes if none were provided");
}
public override void Run()
{
using (var ch = Host.Start(LoadName))
using (var server = InitServer(ch))
{
var settings = CmdParser.GetSettings(Host, ImplOptions, new Arguments());
string cmd = string.Format("maml.exe {0} {1}", LoadName, settings);
ch.Info(cmd);
SendTelemetry(Host);
using (new TimerScope(Host, ch))
{
RunCore(ch, cmd);
}
}
}
protected override void GenerateMethodSignature(IndentingTextWriter w, string prefix,
ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("public static Tuple<IDataView, DataTransform> Create{0}{1}(", prefix, component.LoadNames[0]);
using (w.Nest())
{
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
w.WriteLine("[DataLabInputPort(FriendlyName = \"IDataView\", DisplayName = \"IDataView\", IsOptional = false, DataTypes = WellKnownDataTypeIds.IDataViewDotNet, Description = \"Input data (IDataView)\")]");
w.Write("IDataView data");
var pre = ",";
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateMethodSignature(w, arg, null, null, null, ref pre, "");
}
w.WriteLine(")");
}
}
protected override void GenerateImplCall(IndentingTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
var className = prefix + component.LoadNames[0];
w.WriteLine("var builder = new {0}();", className);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
var arguments = argumentInfo.Args.Where(a => !a.IsHidden).ToArray();
foreach (var arg in arguments)
{
GenerateImplCall(w, arg, "");
}
w.WriteLine("var learner = builder.GetTlcSettings();");
w.WriteLine("return new TlcTrainer(learner.Item1, learner.Item2);");
}
w.WriteLine("}");
}
public override string ToString()
{
if (IsEmpty)
{
return("{}");
}
if (_settings.Length == 0)
{
return(_kind);
}
string str = CmdParser.CombineSettings(_settings);
StringBuilder sb = new StringBuilder();
CmdQuoter.QuoteValue(str, sb, true);
return(_kind + sb.ToString());
}
protected override void GenerateImplCall(IndentedTextWriter w, string prefix, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("{");
using (w.Nest())
{
var className = prefix + component.LoadNames[0];
w.WriteLine("var builder = new {0}();", className);
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateImplCall(w, arg, "");
}
w.WriteLine("var env = new LocalEnvironment(1, verbose: true);");
w.WriteLine("var view = builder.Create{0}{1}Impl(env, data);", prefix, component.LoadNames[0]);
w.WriteLine("return new Tuple<IDataView, DataTransform>(view, new DataTransform(view));");
}
w.WriteLine("}");
}
public void TestPcaTransform()
{
// Force Microsoft.ML.Runtime.PCA assembly to be loaded into the AppDomain so
// ReflectionUtils.FindClassCore does not return null when called by ReflectionUtils.CreateInstance
Assert.AreEqual(typeof(PCAPredictor).Name, "PCAPredictor");
string trainData = GetDataPath(TestDatasets.mnistTiny28.trainFilename);
string fileName = TestContext.TestName + "-Train.txt";
string outFile = DeleteOutputPath(Dir, fileName);
const int rank = 3;
string pcaTransformArgs = string.Format("/inst Trans{{trans=pca {{k={0} seed=1}}}}", rank);
var argsStr1 = string.Format(
"/c CreateInstances {0} {1} /rs=1 /cifile={2}",
trainData,
pcaTransformArgs,
outFile);
var args1 = new TLCArguments();
Assert.IsTrue(CmdParser.ParseArguments(argsStr1, args1));
RunExperiments.Run(args1);
CheckEquality(Dir, fileName);
// Verify the scales of the transformed features decrease with respect to the feature index
TlcTextInstances outputInstances = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile);
Double[] l1norms = new Double[rank];
foreach (Instance instance in outputInstances)
{
Assert.IsTrue(instance.Features.Count == rank);
for (int i = 0; i < instance.Features.Values.Length; i++)
{
l1norms[i] += (instance.Features.Values[i] < 0 ? -instance.Features.Values[i] : instance.Features.Values[i]);
}
}
for (int i = 0; i < l1norms.Length - 1; i++)
{
Assert.IsTrue(l1norms[i] > l1norms[i + 1]);
}
Done();
}
public void TestCreateTextInstancesWithNormalization()
{
TestDataset dataset = TestDatasets.mnistTiny28;
string trainData = GetDataPath(dataset.trainFilename);
string testData = GetDataPath(dataset.testFilename);
var prefix = TestContext.TestName + "-";
string outFile1 = DeleteOutputPath(Dir, prefix + "Norm-Separate-Train.txt");
string outTestFile1 = DeleteOutputPath(Dir, prefix + "Norm-Separate-Test.txt");
string outFile2 = DeleteOutputPath(Dir, prefix + "Norm-Trans-Train.txt");
string outTestFile2 = DeleteOutputPath(Dir, prefix + "Norm-Trans-Test.txt");
string transArgs = "inst=Trans{trans=RFF {rng=1}}";
var argsStr1 = string.Format(
"/c=CreateInstances {0} /test={1} /norm=MinMaxNormalizer /{2} /cifile={3} /citestfile={4}",
trainData, testData, transArgs, outFile1, outTestFile1);
var args1 = new TLCArguments();
Assert.IsTrue(CmdParser.ParseArguments(argsStr1, args1));
RunExperiments.Run(args1);
var argsStr2 = string.Format(
"/c=CreateInstances {0} /test={1} /inst Trans{{trans=MinMaxNormalizer {2}}} /cifile={3} /citestfile={4}",
trainData, testData, transArgs, outFile2, outTestFile2);
var args2 = new TLCArguments();
Assert.IsTrue(CmdParser.ParseArguments(argsStr2, args2));
RunExperiments.Run(args2);
var instances1 = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile1);
var instances2 = new TlcTextInstances(new TlcTextInstances.Arguments(), outFile2);
CompareInstances(instances1, instances2);
var testInstances1 = new TlcTextInstances(new TlcTextInstances.Arguments(), outTestFile1);
var testInstances2 = new TlcTextInstances(new TlcTextInstances.Arguments(), outTestFile2);
CompareInstances(testInstances1, testInstances2);
Done();
}
public override bool Parse(string cmd)
{
var parser = new CmdParser(cmd);
switch (parser.Command)
{
case "exit":
fileManager?.Dispose();
return(false);
case "gc":
GC.Collect();
break;
case "close":
fileManager?.Dispose();
break;
case "make":
MakeDatabase(parser);
break;
case "open":
LoadDatabase(parser);
break;
case "adddata":
AddData(parser);
break;
case "getdata":
GetData(parser);
break;
case "vacuum":
fileManager.DataVacuum();
break;
case "trace":
Trace(parser);
break;
}
return(true);
}
public void TestCreateTextInstances()
{
TestDataset adult = TestDatasets.adult;
string trainData = GetDataPath(adult.trainFilename);
string testData = GetDataPath(adult.testFilename);
var prefix = TestContext.TestName + "-";
string outName = prefix + "Adult-Train.txt";
string statsName = prefix + "Adult-Train.stats.txt";
string outTestName = prefix + "Adult-Test.txt";
string testStatsName = prefix + "Adult-Test.stats.txt";
string outValidName = prefix + "Adult-Valid.txt";
string validStatsName = prefix + "Adult-Valid.stats.txt";
string outFile = DeleteOutputPath(Dir, outName);
string statsFile = DeleteOutputPath(Dir, statsName);
string outTestFile = DeleteOutputPath(Dir, outTestName);
string testStatsFile = DeleteOutputPath(Dir, testStatsName);
string outValidFile = DeleteOutputPath(Dir, outValidName);
string validStatsFile = DeleteOutputPath(Dir, validStatsName);
var argsStr =
string.Format(
"/c=CreateInstances {0} /test={1} /valid={1} /cacheinst=- {2} " +
"/cifile={3} /cistatsfile={4} /citestfile={5} /citeststatsfile={6} /civalidfile={7} /civalidstatsfile={8}",
trainData, testData, adult.extraSettings,
outFile, statsFile, outTestFile, testStatsFile, outValidFile, validStatsFile);
argsStr += " /writer TextInstanceWriter{/stats=+} /disableTracking=+";
var args = new TLCArguments();
Assert.IsTrue(CmdParser.ParseArguments(argsStr, args));
RunExperiments.Run(args);
CheckEquality(Dir, outName);
CheckEquality(Dir, statsName);
CheckEquality(Dir, outTestName);
CheckEquality(Dir, testStatsName);
CheckEquality(Dir, outValidName);
CheckEquality(Dir, validStatsName);
Done();
}
public void Trace(CmdParser parser)
{
var type = parser.GetAttribute("type") ?? parser.GetAttribute(0);
switch (type)
{
case "d":
Console.WriteLine(fileManager.TraceDirs());
break;
case "f":
Console.WriteLine(fileManager.TraceFiles());
break;
default:
Console.WriteLine(fileManager);
break;
}
}
public void WriteTo(CmdParser parser)
{
var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0);
var outFilePath = parser.GetAttribute("out") ?? parser.GetAttribute(1);
//var parent = parser.GetAttribute("parent") ?? parser.GetAttribute(2) ?? "/";
if (fullPath.Substring(0, 1).Equals(":"))
{
var id = fullPath.TrimStart(':').ToInt();
//fileManager.WriteToFile(id, outFilePath);
fileManager.WriteToFile(id, outFilePath);
}
else
{
fileManager.WriteToFile(fullPath, outFilePath);
//fileManager.WriteToFile(fullPath, outFilePath);
}
}
protected void GenerateSummaryComment(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix)
{
if (Exclude.Contains(arg.LongName))
{
return;
}
if (arg.IsSubComponentItemType)
{
// We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer.
Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>));
var types = arg.ItemType.GetGenericArguments();
var baseType = types[0];
var sigType = types[1];
if (sigType == typeof(SignatureDataLoader))
{
return;
}
GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText);
if (IsTrainer(sigType))
{
return;
}
var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType);
foreach (var info in infos)
{
var args = info.CreateArguments();
if (args == null)
{
continue;
}
var argInfo = CmdParser.GetArgInfo(args.GetType(), args);
foreach (var a in argInfo.Args)
{
GenerateSummaryComment(w, a, argSuffix + info.LoadNames[0]);
}
}
}
else
{
GenerateParameterComment(w, arg.LongName + argSuffix, arg.HelpText);
}
}
private static TRes CreateCore <TRes>(
IHostEnvironment env,
Type factoryType,
Type signatureType,
string settings,
out string loadName,
params object[] extraArgs)
where TRes : class
{
Contracts.AssertValue(env);
env.AssertValue(factoryType);
env.AssertValue(signatureType);
env.AssertValue(settings, "settings");
var factory = CmdParser.CreateComponentFactory(factoryType, signatureType, settings);
loadName = factory.Name;
return(ComponentCatalog.CreateInstance <TRes>(env, factory.SignatureType, factory.Name, factory.GetSettingsString(), extraArgs));
}
public override void Run()
{
string command = "Train";
using (var ch = Host.Start(command))
using (var server = InitServer(ch))
{
var settings = CmdParser.GetSettings(Host, Args, new Arguments());
string cmd = string.Format("maml.exe {0} {1}", command, settings);
ch.Info(cmd);
SendTelemetry(Host);
using (new TimerScope(Host, ch))
{
RunCore(ch, cmd);
}
}
}
public void TestInvalidCommandLineArguments()
{
using (StringWriter sw = new StringWriter())
{
Console.SetOut(sw);
CmdParser cmdParser = this.SpawnParser();
string[] args = new string[]
{
"t1=\"in 1\"", "/t2", "/t3=\"in 3\""
};
bool result = cmdParser.ParseArgs(args);
Assert.IsFalse(result, "Invalid argument should return false on ParseArgs");
cmdParser.PrintErrors();
string expected = string.Format(" Invalid command line argument: t1=\"in 1\"{0}", Environment.NewLine);
Assert.AreEqual <string>(expected, sw.ToString(), " Invalid command line argument: t1=\"in 1\"");
}
}
static void Main(string[] args)
{
try
{
var arguments = CmdParser.Parse(args);
var output = InputParser.Parse(arguments.FilePath);
output = FilterHandler.Filter(output, arguments);
OutputPrinter.Print(output);
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.ReadKey();
}
}
public void GetFiles(CmdParser parser)
{
var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0);
FileStructure[] fileStructures;
if (fullPath.Substring(0, 1).Equals(":"))
{
var id = fullPath.TrimStart(':').ToInt();
fileStructures = fileManager.GetFiles(id);
}
else
{
fileStructures = fileManager.GetFiles(fullPath);
}
foreach (var file in fileStructures)
{
Console.WriteLine("{0}", file);
}
}
public void GetDirs(CmdParser parser)
{
var fullPath = parser.GetAttribute("name") ?? parser.GetAttribute(0);
DirectoryStructure[] directoryStructures;
if (fullPath.Substring(0, 1).Equals(":"))
{
var id = fullPath.TrimStart(':').ToInt();
directoryStructures = fileManager.GetDirectories(id);
}
else
{
directoryStructures = fileManager.GetDirectories(fullPath);
}
foreach (var dir in directoryStructures)
{
Console.WriteLine("{0}", dir);
}
}
public void TestMissingCommandLineArguments()
{
using (StringWriter sw = new StringWriter())
{
Console.SetOut(sw);
CmdParser cmdParser = this.SpawnParser();
string[] args = new string[]
{
"/t1=\"in 1\"", "/t2",
};
bool result = cmdParser.ParseArgs(args);
Assert.IsFalse(result, "Missing argument should return false on ParseArgs");
cmdParser.PrintErrors();
string expected = string.Format(" Missing command line argument: t3{0}", Environment.NewLine);
Assert.AreEqual <string>(expected, sw.ToString(), "Missing arguments should provide accurate message");
}
}
public void NewTest()
{
var ml = new MLContext();
ml.AddStandardComponents();
var classes = Utils.MarshalInvoke(ml.ComponentCatalog.FindLoadableClasses <int>, typeof(SignatureCalibrator));
foreach (var cls in classes)
{
var factory = CmdParser.CreateComponentFactory(typeof(IComponentFactory <ICalibratorTrainer>), typeof(SignatureCalibrator), cls.LoadNames[0]);
var calibrator = ((IComponentFactory <ICalibratorTrainer>)factory).CreateComponent(ml);
}
var components = ml.ComponentCatalog.GetAllComponents(typeof(ICalibratorTrainerFactory));
foreach (var component in components)
{
var factory = CmdParser.CreateComponentFactory(typeof(IComponentFactory <ICalibratorTrainer>), typeof(SignatureCalibrator), component.Aliases[0]);
var calibrator = ((IComponentFactory <ICalibratorTrainer>)factory).CreateComponent(ml);
}
}
/// <summary>
/// Creates a data scorer from the 'LoadName{settings}' string.
/// </summary>
/// <param name="env">The host environment to use.</param>
/// <param name="settings">The settings string.</param>
/// <param name="data">The data to score.</param>
/// <param name="predictor">The predictor to score.</param>
/// <param name="trainSchema">The training data schema from which the scorer can optionally extract
/// additional information, for example, label names. If this is <c>null</c>, no information will be
/// extracted.</param>
/// <returns>The scored data.</returns>
internal static IDataScorerTransform CreateScorer(this IHostEnvironment env, string settings,
RoleMappedData data, IPredictor predictor, RoleMappedSchema trainSchema = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(data, nameof(data));
env.CheckValue(predictor, nameof(predictor));
env.CheckValueOrNull(trainSchema);
Type factoryType = typeof(IComponentFactory <IDataView, ISchemaBoundMapper, RoleMappedSchema, IDataScorerTransform>);
Type signatureType = typeof(SignatureDataScorer);
ICommandLineComponentFactory scorerFactorySettings = CmdParser.CreateComponentFactory(
factoryType,
signatureType,
settings);
var bindable = ScoreUtils.GetSchemaBindableMapper(env, predictor, scorerFactorySettings: scorerFactorySettings);
var mapper = bindable.Bind(env, data.Schema);
return(CreateCore <IDataScorerTransform>(env, factoryType, signatureType, settings, data.Data, mapper, trainSchema));
}
// [TestCase("blablabla")]
public void UpdateNonExistingTableTest(string nonExistingTableId)
{
string updateFileName = "filetoupdate" + DateTime.Now.Ticks + ".csv";
string[] args = new string[] { "write-table", nonExistingTableId, updateFileName, "-t", testToken };
//creates test file
FileInfo updateFileInfo = new FileInfo(updateFileName);
string[] dataToUpdate = PrepareDataToUpdate(100);
System.IO.File.WriteAllLines(updateFileInfo.FullName, dataToUpdate);
var command = CmdParser.ParseArguments(args);
Controller ctrl = new Controller(command);
bool result = ctrl.Execute();
Assert.That(result, Is.False);
updateFileInfo.Delete();
}
public void TestCommandLineParserUsingMaxVersion()
{
var args = new[] { "--mode", "export-list", "--db-server", "name", "--rule-names", "rName1" };
var result = CmdParser.Parse(args);
var same = new Args
{
Mode = Mode.ExportList,
BreRecords = new List <BreRecord>()
};
var rule = new BreRecord()
{
UseMaxVersion = true
};
same.RuleNames.Add(rule);
Assert.AreEqual(result.Mode, same.Mode);
for (var i = 0; i < result.RuleNames.Count; i++)
{
Assert.AreEqual(result.RuleNames[i].UseMaxVersion, same.RuleNames[i].UseMaxVersion);
}
}
protected override void GenerateSummaryComment(IndentedTextWriter w, ComponentCatalog.LoadableClassInfo component)
{
w.WriteLine("/// <summary>");
var desc = component.Summary ?? component.LoadNames[0];
using (var sr = new StringReader(desc))
{
string line;
while ((line = sr.ReadLine()) != null)
{
w.WriteLine("/// {0}", line);
}
}
w.WriteLine("/// </summary>");
GenerateParameterComment(w, "data", "The data");
var argumentInfo = CmdParser.GetArgInfo(component.ArgType, component.CreateArguments());
foreach (var arg in argumentInfo.Args.Where(a => !a.IsHidden))
{
GenerateSummaryComment(w, arg, "");
}
}
