public void TestMamlCodeGen()
{
var methodName = System.Reflection.MethodBase.GetCurrentMethod().Name;
var data = FileHelper.GetTestFile("data_train_test.csv");
var output = FileHelper.GetOutputFile("model.zip", methodName);
var bout = new StringBuilder();
var berr = new StringBuilder();
ILogWriter stout = new LogWriter(s => bout.Append(s));
ILogWriter sderr = new LogWriter(s => berr.Append(s));
var cmd = "chain cmd=train{\n" +
"data = __INPUT__\n" +
"loader = text{col=ItemID:I8:0 col=Sentiment:R4:1 col=SentimentSource:TX:2 \n" +
" col=SentimentText:TX:3 col=RowNum:R4:4 \n" +
" col=Label:BL:5 col=Train:BL:6 col=Small:BL:7 header=+ sep=,}\n" +
"xf = concat {col=Features:RowNum,Sentiment}\n" +
"tr = FastTreeBinaryClassification{iter=2}\n" +
"out = __OUTPUT__} \n" +
"cmd = codegen{in=__OUTPUT__ cs=ft_sentiment_cs}";
cmd = cmd.Replace("__INPUT__", data);
cmd = cmd.Replace("__OUTPUT__", output);
using (var env = new DelegateEnvironment(outWriter: stout, errWriter: sderr, verbose: 3))
{
MamlHelper.MamlScript(cmd, false, env);
var sout = bout.ToString();
Assert.IsTrue(sout.Length > 0);
Assert.IsTrue(!sout.Contains("Unknown"));
}
}
/// <summary>
/// Runs a script. Can change the level of desired information.
/// </summary>
/// <param name="script">script to run</param>
/// <param name="catch_output">catches standard output</param>
/// <param name="verbose">2 is default</param>
/// <returns>standard outputs</returns>
public static string MamlScriptConsole(string script, bool catch_output, int verbose = 2)
{
ILogWriter logout = new LogWriter((string s) =>
{
// if (verbose <= 2 && s.Contains("Elapsed"))
// throw new Exception(s);
Console.Write(s);
});
ILogWriter logerr = new LogWriter((string s) =>
{
// if (verbose <= 2 && s.Contains("Elapsed"))
// throw new Exception(s);
if (s.Contains("Elapsed"))
{
Console.Write(s);
}
else
{
Console.Error.Write(s);
}
});
using (var env = new DelegateEnvironment(verbose: verbose, outWriter: logout, errWriter: logerr))
return(MamlScript(script, catch_output, env));
}
/// <summary>
/// Runs a command line with ML.Net.
/// </summary>
/// <param name="script">script to run</param>
/// <param name="catch_output">capture output</param>
/// <param name="env">delegate environment</param>
/// <returns>output and error if captured</returns>
public static string MamlScript(string script, bool catch_output, DelegateEnvironment env = null)
{
int errCode;
string res;
if (catch_output)
{
using (var capture = new StdCapture())
{
errCode = DocumentationEnvironmentHelper.MainWithProgress(script, env);
var sout = capture.StdOut;
var serr = capture.StdErr;
if (string.IsNullOrEmpty(serr))
{
res = sout;
}
else
{
res = $"--OUT--\n{sout}\n--ERR--\n{serr}";
}
}
}
else
{
errCode = DocumentationEnvironmentHelper.MainWithProgress(script, env);
res = string.Empty;
}
if (errCode != 0)
{
throw new MamlException($"Unable to run script, error code={errCode}\n{script}" + (string.IsNullOrEmpty(res) ? string.Empty : $"\n{res}"));
}
return(res);
}
public void TestBcLrSameModel()
{
var methodName = System.Reflection.MethodBase.GetCurrentMethod().Name;
var output = FileHelper.GetOutputFile("bc-lr.zip", methodName);
var name = FileHelper.GetOutputFile("bc.txt", methodName);
var df = DataFrameIO.ReadStr("Label,X1,X2,X3,X4,X5,X6,X7,X8,X9\n" +
"0,0.1,1.1,2.1,3.1,4.1,5.1,6.2,7.4,-5\n" +
"1,1.1,1.1,2.1,3.1,4.1,5.1,6.2,7.4,-5\n" +
"0,2.1,1.1,3.1,3.1,-4.1,5.1,6.2,7.4,-5\n" +
"1,3.1,1.1,4.1,3.1,4.1,-5.1,6.2,7.4,-5\n" +
"0,4.1,1.1,2.1,3.1,4.1,5.1,6.2,-7.4,-5");
df.ToCsv(name);
var cmd = string.Format("Train tr=lr data={0} out={1} loader=text{{col=Label:R4:0 col=Features:R4:1-* sep=, header=+}}",
name, output);
var stdout = new StringBuilder();
ILogWriter logout = new LogWriter((string s) => { stdout.Append(s); });
ILogWriter logerr = new LogWriter((string s) => { stdout.Append(s); });
/*using (*/
var env = new DelegateEnvironment(seed: 0, verbose: 2, outWriter: logout, errWriter: logerr);
MamlHelper.MamlScript(cmd, false, env);
var stout = stdout.ToString();
if (string.IsNullOrEmpty(stout))
{
throw new Exception(stout);
}
}
/// <summary>
/// Runs a script.
/// </summary>
/// <param name="args">script or command line arguments</param>
/// <param name="env">environment, created if null</param>
/// <returns>exit code</returns>
public static int MainWithProgress(string args, DelegateEnvironment env = null)
{
string currentDirectory = Path.GetDirectoryName(typeof(Maml).Module.FullyQualifiedName);
//bool dispose = false;
var keepOut = new StringBuilder();
var keepErr = new StringBuilder();
if (env == null)
{
ILogWriter logout = new LogWriter((string s) =>
{
keepOut.Append(s);
//if (env.VerboseLevel <= 2 && s.Contains("Elapsed"))
// throw new Exception(string.Format("{0}\n---\n{1}", s, keepOut.ToString()));
Console.Write(s);
});
ILogWriter logerr = new LogWriter((string s) =>
{
keepErr.Append(s);
if (env.VerboseLevel <= 2 && s.Contains("Elapsed"))
{
// We do nothing.
}
if (s.Contains("Elapsed"))
{
Console.Write(s);
}
else
{
Console.Error.Write(s);
}
});
env = new DelegateEnvironment(seed: 0, verbose: 2, outWriter: logout, errWriter: logerr);
//dispose = true;
}
int ret = 0;
using (var progressCancel = new CancellationTokenSource())
using (CreateAssemblyRegistrar(env, currentDirectory))
{
var progressTrackerTask = Task.Run(() => TrackProgress(env, progressCancel.Token));
try
{
ret = MainCore(env, args, true);
}
finally
{
progressCancel.Cancel();
progressTrackerTask.Wait();
// If the run completed so quickly that the progress task was cancelled before it even got a chance to start,
// we need to gather the checkpoints.
env.PrintProgress();
}
}
//if (dispose)
//env.Dispose();
return(ret);
}
public void TestScikitAPI_DelegateEnvironment()
{
var inputs = new[] {
new ExampleA()
{
X = new float[] { 1, 10, 100 }
},
new ExampleA()
{
X = new float[] { 2, 3, 5 }
}
};
var inputs2 = new[] {
new ExampleA()
{
X = new float[] { -1, -10, -100 }
},
new ExampleA()
{
X = new float[] { -2, -3, -5 }
}
};
var stdout = new List <string>();
var stderr = new List <string>();
ILogWriter logout = new LogWriter((string s) =>
{
stdout.Add(s);
});
ILogWriter logerr = new LogWriter((string s) =>
{
stderr.Add(s);
});
using (var host = new DelegateEnvironment(conc: 1, outWriter: logout, errWriter: logerr, verbose: 3))
using (var ch = host.Start("Train Pipeline"))
{
ComponentHelper.AddStandardComponents(host);
ch.Info(MessageSensitivity.All, "Polynomial");
var data = host.CreateStreamingDataView(inputs);
using (var pipe = new ScikitPipeline(new[] { "poly{col=X}" }, host: host))
{
var predictor = pipe.Train(data);
if (predictor == null)
{
throw new Exception("Predictor is null");
}
}
}
if (stdout.Count == 0)
{
throw new Exception("stdout is empty.");
}
if (stderr.Count != 0)
{
throw new Exception($"stderr not empty\n{string.Join("\n", stderr)}");
}
}
public static IHostEnvironment CreateEnvironment(int seed = -1, int verbose = 0,
string sensitivity = "All", int conc = 0,
ILogWriter outWriter = null, ILogWriter errWriter = null)
{
var env = new DelegateEnvironment(seed: seed < 0 ? null : (int?)seed, verbose: verbose,
sensitivity: MessageSensitivityFromString(sensitivity),
conc: conc, outWriter: outWriter, errWriter: errWriter);
ComponentHelper.AddStandardComponents(env);
return(env);
}
public void TestHelpScorer()
{
var bout = new StringBuilder();
var berr = new StringBuilder();
ILogWriter stout = new LogWriter(s => bout.Append(s));
ILogWriter sderr = new LogWriter(s => berr.Append(s));
using (var env = new DelegateEnvironment(outWriter: stout, errWriter: sderr, verbose: 3))
{
var cmd = "? MultiClassClassifierScorer";
MamlHelper.MamlScript(cmd, false, env);
var sout = bout.ToString();
Assert.IsTrue(sout.Length > 0);
Assert.IsTrue(!sout.Contains("Unknown"));
}
}
public static Tuple <IHostEnvironment, StringBuilder, StringBuilder> CreateStoreEnvironment(int seed = -1, int verbose = 0,
string sensitivity = "All", int conc = 0)
{
ILogWriter logout, logerr;
var stout = new StringBuilder();
var sterr = new StringBuilder();
logout = new LogWriter((string s) => { stout.Append(s); });
logerr = new LogWriter((string s) => { sterr.Append(s); });
var env = new DelegateEnvironment(seed: seed < 0 ? null : (int?)seed, verbose: verbose,
sensitivity: MessageSensitivityFromString(sensitivity),
conc: conc, outWriter: logout, errWriter: logerr);
ComponentHelper.AddStandardComponents(env);
return(new Tuple <IHostEnvironment, StringBuilder, StringBuilder>(env, stout, sterr));
}
public void TestScikitAPI_DelegateEnvironmentVerbose0()
{
var inputs = new[] {
new ExampleA()
{
X = new float[] { 1, 10, 100 }
},
new ExampleA()
{
X = new float[] { 2, 3, 5 }
}
};
var inputs2 = new[] {
new ExampleA()
{
X = new float[] { -1, -10, -100 }
},
new ExampleA()
{
X = new float[] { -2, -3, -5 }
}
};
var stdout = new List <string>();
var stderr = new List <string>();
ILogWriter logout = new LogWriter(s => stdout.Add(s));
ILogWriter logerr = new LogWriter(s => stderr.Add(s));
/*using (*/
var host = new DelegateEnvironment(seed: 0, outWriter: logout, errWriter: logerr, verbose: 0);
{
ComponentHelper.AddStandardComponents(host);
var data = DataViewConstructionUtils.CreateFromEnumerable(host, inputs);
using (var pipe = new ScikitPipeline(new[] { "poly{col=X}" }, "km{k=2}", host: host))
{
var predictor = pipe.Train(data, feature: "X");
Assert.IsTrue(predictor != null);
}
}
Assert.AreEqual(stdout.Count, 0);
Assert.AreEqual(stderr.Count, 0);
}
public void TestHelpModels()
{
foreach (var name in new[] { "Resample" })
{
var bout = new StringBuilder();
var berr = new StringBuilder();
ILogWriter stout = new LogWriter(s => bout.Append(s));
ILogWriter sderr = new LogWriter(s => berr.Append(s));
using (var env = new DelegateEnvironment(outWriter: stout, errWriter: sderr, verbose: 3))
{
var cmd = $"? {name}";
MamlHelper.MamlScript(cmd, false, env: env);
var sout = bout.ToString();
var serr = berr.ToString();
Assert.IsTrue(!serr.Contains("Can't instantiate"));
Assert.IsTrue(sout.Length > 0);
Assert.IsTrue(!sout.Contains("Unknown"));
}
}
}
private static void TrackProgress(DelegateEnvironment env, CancellationToken ct)
{
try
{
while (!ct.IsCancellationRequested)
{
TimeSpan interval = TimeSpan.FromSeconds(0.6);
if (ct.WaitHandle.WaitOne(interval))
{
return;
}
env.PrintProgress();
}
}
catch (Exception ex)
{
Console.Error.WriteLine("Progress tracking terminated with an exception");
PrintExceptionData(Console.Error, ex, false);
Console.Error.WriteLine("Progress tracking is terminated.");
}
}
public void TestCSGenerator()
{
var methodName = System.Reflection.MethodBase.GetCurrentMethod().Name;
var basePath = FileHelper.GetOutputFile("CSharpApiExt.cs", methodName);
var cmd = $"? generator=cs{{csFilename={basePath} exclude=System.CodeDom.dll}}";
var bout = new StringBuilder();
var berr = new StringBuilder();
ILogWriter stout = new LogWriter(s => bout.Append(s));
ILogWriter sderr = new LogWriter(s => berr.Append(s));
using (var env = new DelegateEnvironment(outWriter: stout, errWriter: sderr, verbose: 3))
{
MamlHelper.MamlScript(cmd, false, env: env);
var sout = bout.ToString();
var serr = berr.ToString();
Assert.IsTrue(sout.Length > 0);
Assert.IsTrue(serr.Length == 0);
Assert.IsFalse(sout.ToLower().Contains("usage"));
}
var text = File.ReadAllText(basePath);
Assert.IsTrue(text.ToLower().Contains("nearest"));
}
static int MainCore(DelegateEnvironment env, string args, bool alwaysPrintStacktrace)
{
var mainHost = env.Register("Main");
using (var ch = mainHost.Start("Main"))
{
int result;
try
{
if (!CmdParser.TryGetFirstToken(args, out string kind, out string settings))
{
Usage();
return(-1);
}
if (!ComponentCatalog.TryCreateInstance <ICommand, SignatureCommand>(mainHost, out ICommand cmd, kind, settings))
{
ch.Error("Unknown command: '{0}'", kind);
Usage();
return(-1);
}
var helpC = cmd as HelpCommand;
if (helpC == null)
{
env.SetPrintElapsed(true);
cmd.Run();
}
else
{
int width = 80;
try
{
width = Console.BufferWidth;
}
catch (Exception)
{
}
env.SetPrintElapsed(false);
helpC.Run(width);
}
result = 0;
}
catch (Exception ex)
{
int count = 0;
for (var e = ex; e != null; e = e.InnerException)
{
// Telemetry: Log the exception
if (e.IsMarked())
{
ch.Error(e.Sensitivity(), e.Message);
PrintExceptionData(ch, e, false);
count++;
}
}
if (count == 0)
{
ch.Error(MessageSensitivity.None, "Unexpected failure.");
}
if (count == 0 || alwaysPrintStacktrace)
{
ch.Error(MessageSensitivity.None, "===== Begin detailed dump =====");
PrintFullExceptionDetails(ch, ex);
if (env.VerboseLevel >= 3)
{
ch.Error("= LoadedAssemblies =");
var assemblies = AppDomain.CurrentDomain.GetAssemblies().Select(x => InfoAssembly(x)).OrderBy(c => c);
foreach (var a in assemblies)
{
ch.Error(a);
}
}
ch.Error(MessageSensitivity.None, "====== End detailed dump =====");
}
// Return a negative result code so AEther recognizes this as a failure.
result = count > 0 ? -1 : -2;
}
finally
{
}
return(result);
}
}
public static ScikitOnnxContext ToOnnx(IDataTransform trans, ref string[] inputs, ref string[] outputs,
string name = null, string producer = "Scikit.ML",
long version = 0, string domain = "onnx.ai.ml",
OnnxVersion onnxVersion = OnnxVersion.Stable,
IDataView[] begin = null, IHostEnvironment host = null)
{
if (host == null)
{
using (var env = new DelegateEnvironment())
return(ToOnnx(trans, ref inputs, ref outputs, name, producer, version, domain, onnxVersion, begin, env));
}
if (name == null)
{
name = trans.GetType().Name;
}
GuessOutputs(trans, ref outputs);
GuessInputs(trans, ref inputs, begin);
if (inputs == null || inputs.Length == 0)
{
throw host.Except("Inputs cannot be empty.");
}
if (outputs == null || outputs.Length == 0)
{
throw host.Except("Outputs cannot be empty.");
}
var assembly = System.Reflection.Assembly.GetExecutingAssembly();
var versionInfo = System.Diagnostics.FileVersionInfo.GetVersionInfo(assembly.Location);
var ctx = new ScikitOnnxContext(host, name, producer, versionInfo.FileVersion,
version, domain, onnxVersion);
var hasin = new HashSet <string>(inputs);
var uniqueVars = EnumerateVariables(trans, begin).ToArray();
var mapInputType = new Dictionary <string, ColumnType>();
foreach (var it in uniqueVars.Where(c => c.position == TagHelper.GraphPositionEnum.first))
{
mapInputType[it.variableName] = it.variableType;
}
foreach (var col in inputs)
{
ctx.AddInputVariable(mapInputType[col], col);
}
var views = TagHelper.EnumerateAllViews(trans, begin);
var transforms = views.Where(c => (c.Item1 as IDataTransform) != null)
.Select(c => c.Item1)
.ToArray();
foreach (var tr in transforms.Reverse())
{
var tron = tr as ICanSaveOnnx;
if (tron == null)
{
throw host.ExceptNotSupp($"Transform {tr.GetType()} cannot be saved in Onnx format.");
}
if (!tron.CanSaveOnnx(ctx))
{
throw host.ExceptNotSupp($"Transform {tr.GetType()} cannot be saved in ONNX format.");
}
var tron2 = tron as ISaveAsOnnx;
if (!tron2.CanSaveOnnx(ctx))
{
throw host.ExceptNotSupp($"Transform {tr.GetType()} does not implement SaveAsOnnx.");
}
tron2.SaveAsOnnx(ctx);
}
var mapOuputType = new Dictionary <string, ColumnType>();
foreach (var it in uniqueVars.Where(c => c.position == TagHelper.GraphPositionEnum.last))
{
mapOuputType[it.variableName] = it.variableType;
}
foreach (var col in outputs)
{
var variableName = ctx.TryGetVariableName(col);
var trueVariableName = ctx.AddIntermediateVariable(null, col, true);
ctx.CreateNode("Identity", variableName, trueVariableName, ctx.GetNodeName("Identity"), "");
ctx.AddOutputVariable(mapOuputType[col], trueVariableName);
}
return(ctx);
}
public static string MamlScript(string script, bool catch_output,
int conc = 0, int verbose = 2, int sensitivity = -1,
bool skipFailLoading = false)
{
ILogWriter logout, logerr;
var stout = new StringBuilder();
var sterr = new StringBuilder();
if (catch_output)
{
logout = new LogWriter((string s) =>
{
stout.Append(s);
});
logerr = new LogWriter((string s) =>
{
sterr.Append(s);
});
}
else
{
logout = new LogWriter((string s) => { Console.Write(s); });
logerr = new LogWriter((string s) => { Console.Error.Write(s); });
}
using (var env = new DelegateEnvironment((int?)null, verbose, (MessageSensitivity)sensitivity, conc, logout, logerr))
{
if (skipFailLoading)
{
try
{
ComponentHelper.AddStandardComponents(env);
}
catch (Exception e)
{
sterr.Append($"Unable to load an assembly due to {e.ToString()}.\n");
}
}
else
{
ComponentHelper.AddStandardComponents(env);
}
string res = null;
try
{
res = MamlHelper.MamlScript(script, false, env);
if (!string.IsNullOrEmpty(res))
{
stout.Append(res);
}
}
catch (InvalidOperationException e)
{
if (!string.IsNullOrEmpty(res))
{
sterr.Append(res + "\n");
}
sterr.Append($"[PyMamlHelper-InvalidOperationException] catch_output={catch_output}\n" + e.ToString() + "\n");
}
catch (MamlException e)
{
if (!string.IsNullOrEmpty(res))
{
sterr.Append(res + "\n");
}
sterr.Append($"[PyMamlHelper-MamlException] catch_output={catch_output}\n" + e.ToString() + "\n");
}
if (sterr.Length > 0)
{
return(string.Format("---OUT---\n{0}\n---ERR---\n{1}", stout.ToString(), sterr.ToString()));
}
else
{
return(stout.ToString());
}
}
}
