/// <summary>
// The Generic entry point. The specific behavior is indicated in a string argument.
/// </summary>
private static unsafe int GenericExec(EnvironmentBlock *penv, sbyte *psz, int cdata, DataSourceBlock **ppdata)
{
var env = new RmlEnvironment(MarshalDelegate <CheckCancelled>(penv->checkCancel), penv->seed, verbose: penv != null && penv->verbosity > 3);
var host = env.Register("ML.NET_Execution");
env.ComponentCatalog.RegisterAssembly(typeof(TextLoader).Assembly); // ML.Data
env.ComponentCatalog.RegisterAssembly(typeof(LinearModelParameters).Assembly); // ML.StandardLearners
env.ComponentCatalog.RegisterAssembly(typeof(CategoricalCatalog).Assembly); // ML.Transforms
env.ComponentCatalog.RegisterAssembly(typeof(FastTreeRegressionTrainer).Assembly); // ML.FastTree
//env.ComponentCatalog.RegisterAssembly(typeof(EnsembleModelParameters).Assembly); // ML.Ensemble
env.ComponentCatalog.RegisterAssembly(typeof(KMeansModelParameters).Assembly); // ML.KMeansClustering
env.ComponentCatalog.RegisterAssembly(typeof(PcaModelParameters).Assembly); // ML.PCA
env.ComponentCatalog.RegisterAssembly(typeof(CVSplit).Assembly); // ML.EntryPoints
env.ComponentCatalog.RegisterAssembly(typeof(OlsModelParameters).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(SymSgdClassificationTrainer).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(AutoInference).Assembly); // ML.PipelineInference
env.ComponentCatalog.RegisterAssembly(typeof(DataViewReference).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(ImageLoadingTransformer).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(SaveOnnxCommand).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(TimeSeriesProcessingEntryPoints).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(ParquetLoader).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(ForecastExtensions).Assembly);
using (var ch = host.Start("Executing"))
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
try
{
// code, pszIn, and pszOut can be null.
ch.Trace("Checking parameters");
host.CheckParam(penv != null, nameof(penv));
host.CheckParam(penv->messageSink != null, "penv->message");
host.CheckParam(psz != null, nameof(psz));
ch.Trace("Converting graph operands");
var graph = BytesToString(psz);
ch.Trace("Wiring message sink");
var message = MarshalDelegate <MessageSink>(penv->messageSink);
var messageValidator = new MessageValidator(host);
var lk = new object();
Action <IMessageSource, ChannelMessage> listener =
(sender, msg) =>
{
byte[] bs = StringToNullTerminatedBytes(sender.FullName);
string m = messageValidator.Validate(msg);
if (!string.IsNullOrEmpty(m))
{
byte[] bm = StringToNullTerminatedBytes(m);
lock (lk)
{
fixed(byte *ps = bs)
fixed(byte *pm = bm)
message(penv, msg.Kind, (sbyte *)ps, (sbyte *)pm);
}
}
};
env.AddListener(listener);
host.CheckParam(cdata >= 0, nameof(cdata), "must be non-negative");
host.CheckParam(ppdata != null || cdata == 0, nameof(ppdata));
for (int i = 0; i < cdata; i++)
{
var pdata = ppdata[i];
host.CheckParam(pdata != null, "pdata");
host.CheckParam(0 <= pdata->ccol && pdata->ccol <= int.MaxValue, "ccol");
host.CheckParam(0 <= pdata->crow && pdata->crow <= long.MaxValue, "crow");
if (pdata->ccol > 0)
{
host.CheckParam(pdata->names != null, "names");
host.CheckParam(pdata->kinds != null, "kinds");
host.CheckParam(pdata->keyCards != null, "keyCards");
host.CheckParam(pdata->vecCards != null, "vecCards");
host.CheckParam(pdata->getters != null, "getters");
}
}
ch.Trace("Validating number of data sources");
// Wrap the data sets.
ch.Trace("Wrapping native data sources");
ch.Trace("Executing");
ExecCore(penv, host, ch, graph, cdata, ppdata);
}
catch (Exception e)
{
// Dump the exception chain.
var ex = e;
while (ex.InnerException != null)
{
ex = ex.InnerException;
}
ch.Error("*** {1}: '{0}'", ex.Message, ex.GetType());
return(-1);
}
finally
{
sw.Stop();
if (penv != null && penv->verbosity > 0)
{
ch.Info("Elapsed time: {0}", sw.Elapsed);
}
else
{
ch.Trace("Elapsed time: {0}", sw.Elapsed);
}
}
}
return(0);
}
private static void ExecCore(EnvironmentBlock *penv, IHost host, IChannel ch, string graph, int cdata, DataSourceBlock **ppdata)
{
Contracts.AssertValue(ch);
ch.AssertValue(host);
ch.AssertNonEmpty(graph);
ch.Assert(cdata >= 0);
ch.Assert(ppdata != null || cdata == 0);
RunGraphCore(penv, host, graph, cdata, ppdata);
}
private static void RunGraphCore(EnvironmentBlock *penv, IHostEnvironment env, string graphStr, int cdata, DataSourceBlock **ppdata)
{
Contracts.AssertValue(env);
var host = env.Register("RunGraph", penv->seed, null);
JObject graph;
try
{
graph = JObject.Parse(graphStr);
}
catch (JsonReaderException ex)
{
throw host.Except(ex, "Failed to parse experiment graph: {0}", ex.Message);
}
var runner = new GraphRunner(host, graph["nodes"] as JArray);
var dvNative = new IDataView[cdata];
try
{
for (int i = 0; i < cdata; i++)
{
dvNative[i] = new NativeDataView(host, ppdata[i]);
}
// Setting inputs.
var jInputs = graph["inputs"] as JObject;
if (graph["inputs"] != null && jInputs == null)
{
throw host.Except("Unexpected value for 'inputs': {0}", graph["inputs"]);
}
int iDv = 0;
if (jInputs != null)
{
foreach (var kvp in jInputs)
{
var pathValue = kvp.Value as JValue;
if (pathValue == null)
{
throw host.Except("Invalid value for input: {0}", kvp.Value);
}
var path = pathValue.Value <string>();
var varName = kvp.Key;
var type = runner.GetPortDataKind(varName);
switch (type)
{
case TlcModule.DataKind.FileHandle:
var fh = new SimpleFileHandle(host, path, false, false);
runner.SetInput(varName, fh);
break;
case TlcModule.DataKind.DataView:
IDataView dv;
if (!string.IsNullOrWhiteSpace(path))
{
var extension = Path.GetExtension(path);
if (extension == ".txt")
{
dv = TextLoader.LoadFile(host, new TextLoader.Options(), new MultiFileSource(path));
}
else if (extension == ".dprep")
{
dv = LoadDprepFile(BytesToString(penv->pythonPath), path);
}
else
{
dv = new BinaryLoader(host, new BinaryLoader.Arguments(), path);
}
}
else
{
Contracts.Assert(iDv < dvNative.Length);
// prefetch all columns
dv = dvNative[iDv++];
var prefetch = new int[dv.Schema.Count];
for (int i = 0; i < prefetch.Length; i++)
{
prefetch[i] = i;
}
dv = new CacheDataView(host, dv, prefetch);
}
runner.SetInput(varName, dv);
break;
case TlcModule.DataKind.PredictorModel:
PredictorModel pm;
if (!string.IsNullOrWhiteSpace(path))
{
using (var fs = File.OpenRead(path))
pm = new PredictorModelImpl(host, fs);
}
else
{
throw host.Except("Model must be loaded from a file");
}
runner.SetInput(varName, pm);
break;
case TlcModule.DataKind.TransformModel:
TransformModel tm;
if (!string.IsNullOrWhiteSpace(path))
{
using (var fs = File.OpenRead(path))
tm = new TransformModelImpl(host, fs);
}
else
{
throw host.Except("Model must be loaded from a file");
}
runner.SetInput(varName, tm);
break;
default:
throw host.Except("Port type {0} not supported", type);
}
}
}
runner.RunAll();
// Reading outputs.
using (var ch = host.Start("Reading outputs"))
{
var jOutputs = graph["outputs"] as JObject;
if (jOutputs != null)
{
foreach (var kvp in jOutputs)
{
var pathValue = kvp.Value as JValue;
if (pathValue == null)
{
throw host.Except("Invalid value for input: {0}", kvp.Value);
}
var path = pathValue.Value <string>();
var varName = kvp.Key;
var type = runner.GetPortDataKind(varName);
switch (type)
{
case TlcModule.DataKind.FileHandle:
var fh = runner.GetOutput <IFileHandle>(varName);
throw host.ExceptNotSupp("File handle outputs not yet supported.");
case TlcModule.DataKind.DataView:
var idv = runner.GetOutput <IDataView>(varName);
if (path == CSR_MATRIX)
{
SendViewToNativeAsCsr(ch, penv, idv);
}
else if (!string.IsNullOrWhiteSpace(path))
{
SaveIdvToFile(idv, path, host);
}
else
{
var infos = ProcessColumns(ref idv, penv->maxSlots, host);
SendViewToNativeAsDataFrame(ch, penv, idv, infos);
}
break;
case TlcModule.DataKind.PredictorModel:
var pm = runner.GetOutput <PredictorModel>(varName);
if (!string.IsNullOrWhiteSpace(path))
{
SavePredictorModelToFile(pm, path, host);
}
else
{
throw host.Except("Returning in-memory models is not supported");
}
break;
case TlcModule.DataKind.TransformModel:
var tm = runner.GetOutput <TransformModel>(varName);
if (!string.IsNullOrWhiteSpace(path))
{
using (var fs = File.OpenWrite(path))
tm.Save(host, fs);
}
else
{
throw host.Except("Returning in-memory models is not supported");
}
break;
case TlcModule.DataKind.Array:
var objArray = runner.GetOutput <object[]>(varName);
if (objArray is PredictorModel[])
{
var modelArray = (PredictorModel[])objArray;
// Save each model separately
for (var i = 0; i < modelArray.Length; i++)
{
var modelPath = string.Format(CultureInfo.InvariantCulture, path, i);
SavePredictorModelToFile(modelArray[i], modelPath, host);
}
}
else
{
throw host.Except("DataKind.Array type {0} not supported", objArray.First().GetType());
}
break;
default:
throw host.Except("Port type {0} not supported", type);
}
}
}
}
}
finally
{
// The raw data view is disposable so it lets go of unmanaged raw pointers before we return.
for (int i = 0; i < dvNative.Length; i++)
{
var view = dvNative[i];
if (view == null)
{
continue;
}
host.Assert(view is IDisposable);
var disp = (IDisposable)dvNative[i];
disp.Dispose();
}
}
}
