private ILegacyDataLoader CreateLoaderFromBytes(byte[] loaderBytes, IMultiStreamSource files)
{
Contracts.CheckValue(loaderBytes, nameof(loaderBytes));
Contracts.CheckValue(files, nameof(files));
using (var stream = new MemoryStream(loaderBytes))
using (var rep = RepositoryReader.Open(stream, _host))
{
return(ModelFileUtils.LoadLoader(_host, rep, files, false));
}
}
public void LoadOriginalBinaryLoaderModel()
{
var env = new MLContext().AddStandardComponents();
using (var modelStream = File.OpenRead(Path.Combine("TestModels", "BinaryLoader-v3.11.0.0.zip")))
using (var rep = RepositoryReader.Open(modelStream, env))
{
IDataLoader result = ModelFileUtils.LoadLoader(env, rep, new MultiFileSource(null), true);
Assert.Equal(2, result.Schema.Count);
Assert.Equal("Image", result.Schema[0].Name);
Assert.Equal("Class", result.Schema[1].Name);
}
}
/// <summary>
/// Load the model and its input schema from the stream.
/// </summary>
/// <param name="stream">A readable, seekable stream to load from.</param>
/// <param name="inputSchema">Will contain the input schema for the model. If the model was saved using older APIs
/// it may not contain an input schema, in this case <paramref name="inputSchema"/> will be null.</param>
/// <returns>The loaded model.</returns>
public ITransformer Load(Stream stream, out DataViewSchema inputSchema)
{
_env.CheckValue(stream, nameof(stream));
using (var rep = RepositoryReader.Open(stream, _env))
{
var entry = rep.OpenEntryOrNull(SchemaEntryName);
if (entry != null)
{
var loader = new BinaryLoader(_env, new BinaryLoader.Arguments(), entry.Stream);
inputSchema = loader.Schema;
ModelLoadContext.LoadModel <ITransformer, SignatureLoadModel>(_env, out var transformerChain, rep,
CompositeDataLoader <object, ITransformer> .TransformerDirectory);
return(transformerChain);
}
ModelLoadContext.LoadModelOrNull <IDataLoader <IMultiStreamSource>, SignatureLoadModel>(_env, out var dataLoader, rep, null);
if (dataLoader == null)
{
// Try to see if the model was saved without a loader or a schema.
if (ModelLoadContext.LoadModelOrNull <ITransformer, SignatureLoadModel>(_env, out var transformerChain, rep,
CompositeDataLoader <object, ITransformer> .TransformerDirectory))
{
inputSchema = null;
return(transformerChain);
}
// Try to load from legacy model format.
try
{
var loader = ModelFileUtils.LoadLoader(_env, rep, new MultiFileSource(null), false);
inputSchema = loader.Schema;
return(TransformerChain.LoadFromLegacy(_env, stream));
}
catch (Exception ex)
{
throw _env.Except(ex, "Could not load legacy format model");
}
}
if (dataLoader is CompositeDataLoader <IMultiStreamSource, ITransformer> composite)
{
inputSchema = composite.Loader.GetOutputSchema();
return(composite.Transformer);
}
inputSchema = dataLoader.GetOutputSchema();
return(new TransformerChain <ITransformer>());
}
}
public void TestTextLoaderKeyTypeBackCompat()
{
// Model generated with the following command on a version of the code previous to the KeyType change that removed Min and Contiguous:
// Train data=...\breast-cancer.txt loader =TextLoader{col=Label:R4:0 col=Features:R4:1-9 col=key:U4[0-*]:3} tr=LogisticRegression {} out=model.zip
var mlContext = new MLContext(1);
string textLoaderModelPath = GetDataPath("backcompat/textloader-with-key-model.zip");
string breastCancerPath = GetDataPath(TestDatasets.breastCancer.trainFilename);
using (FileStream modelfs = File.OpenRead(textLoaderModelPath))
using (var rep = RepositoryReader.Open(modelfs, mlContext))
{
var result = ModelFileUtils.LoadLoader(mlContext, rep, new MultiFileSource(breastCancerPath), false);
Assert.True(result.Schema.TryGetColumnIndex("key", out int featureIdx));
Assert.True(result.Schema[featureIdx].Type is KeyDataViewType keyType && keyType.Count == typeof(uint).ToMaxInt());
}
}
private IDataScorerTransform GetScorer(IHostEnvironment env, IDataView transforms, IPredictor pred, string testDataPath = null)
{
using (var ch = env.Start("Saving model"))
using (var memoryStream = new MemoryStream())
{
var trainRoles = new RoleMappedData(transforms, label: "Label", feature: "Features");
// Model cannot be saved with CacheDataView
TrainUtils.SaveModel(env, ch, memoryStream, pred, trainRoles);
memoryStream.Position = 0;
using (var rep = RepositoryReader.Open(memoryStream, ch))
{
IDataLoader testPipe = ModelFileUtils.LoadLoader(env, rep, new MultiFileSource(testDataPath), true);
RoleMappedData testRoles = new RoleMappedData(testPipe, label: "Label", feature: "Features");
return(ScoreUtils.GetScorer(pred, testRoles, env, testRoles.Schema));
}
}
}
public void TestTextLoaderBackCompat_VerWritt_0x0001000C()
{
// Checks backward compatibility with a text loader created with "verWrittenCur: 0x0001000C"
// Model generated with:
// loader=text{header+ col=SepalLength:Num:0 col=SepalWidth:Num:1 col=PetalLength:Num:2 col=PetalWidth:Num:2 col=Cat:TX:1-8 col=Num:9-14 col=Type:TX:4}
var mlContext = new MLContext(1);
string textLoaderModelPath = GetDataPath("backcompat/textloader_VerWritt_0x0001000C.zip");
string irisPath = GetDataPath(TestDatasets.irisData.trainFilename);
IDataView iris;
using (FileStream modelfs = File.OpenRead(textLoaderModelPath))
using (var rep = RepositoryReader.Open(modelfs, mlContext))
{
iris = ModelFileUtils.LoadLoader(mlContext, rep, new MultiFileSource(irisPath), false);
}
var previewIris = iris.Preview(1);
var irisFirstRow = new Dictionary <string, float>();
irisFirstRow["SepalLength"] = 5.1f;
irisFirstRow["SepalWidth"] = 3.5f;
irisFirstRow["PetalLength"] = 1.4f;
irisFirstRow["PetalWidth"] = 0.2f;
Assert.Equal(5, previewIris.ColumnView.Length);
Assert.Equal("SepalLength", previewIris.Schema[0].Name);
Assert.Equal(NumberDataViewType.Single, previewIris.Schema[0].Type);
int index = 0;
foreach (var entry in irisFirstRow)
{
Assert.Equal(entry.Key, previewIris.RowView[0].Values[index].Key);
Assert.Equal(entry.Value, previewIris.RowView[0].Values[index++].Value);
}
Assert.Equal("Type", previewIris.RowView[0].Values[index].Key);
Assert.Equal("Iris-setosa", previewIris.RowView[0].Values[index].Value.ToString());
}
/// <summary>
/// Loads multiple artifacts of interest from the input model file, given the context
/// established by the command line arguments.
/// </summary>
/// <param name="ch">The channel to which to provide output.</param>
/// <param name="wantPredictor">Whether we want a predictor from the model file. If
/// <c>false</c> we will not even attempt to load a predictor. If <c>null</c> we will
/// load the predictor, if present. If <c>true</c> we will load the predictor, or fail
/// noisily if we cannot.</param>
/// <param name="predictor">The predictor in the model, or <c>null</c> if
/// <paramref name="wantPredictor"/> was false, or <paramref name="wantPredictor"/> was
/// <c>null</c> and no predictor was present.</param>
/// <param name="wantTrainSchema">Whether we want the training schema. Unlike
/// <paramref name="wantPredictor"/>, this has no "hard fail if not present" option. If
/// this is <c>true</c>, it is still possible for <paramref name="trainSchema"/> to remain
/// <c>null</c> if there were no role mappings, or pipeline.</param>
/// <param name="trainSchema">The training schema if <paramref name="wantTrainSchema"/>
/// is true, and there were role mappings stored in the model.</param>
/// <param name="pipe">The data pipe constructed from the combination of the
/// model and command line arguments.</param>
protected void LoadModelObjects(
IChannel ch,
bool?wantPredictor, out IPredictor predictor,
bool wantTrainSchema, out RoleMappedSchema trainSchema,
out ILegacyDataLoader pipe)
{
// First handle the case where there is no input model file.
// Everything must come from the command line.
using (var file = Host.OpenInputFile(ImplOptions.InputModelFile))
using (var strm = file.OpenReadStream())
using (var rep = RepositoryReader.Open(strm, Host))
{
// First consider loading the predictor.
if (wantPredictor == false)
{
predictor = null;
}
else
{
ch.Trace("Loading predictor");
predictor = ModelFileUtils.LoadPredictorOrNull(Host, rep);
if (wantPredictor == true)
{
Host.Check(predictor != null, "Could not load predictor from model file");
}
}
// Next create the loader.
var loaderFactory = ImplOptions.Loader;
ILegacyDataLoader trainPipe = null;
if (loaderFactory != null)
{
// The loader is overridden from the command line.
pipe = loaderFactory.CreateComponent(Host, new MultiFileSource(ImplOptions.DataFile));
if (ImplOptions.LoadTransforms == true)
{
Host.CheckUserArg(!string.IsNullOrWhiteSpace(ImplOptions.InputModelFile), nameof(ImplOptions.InputModelFile));
pipe = LoadTransformChain(pipe);
}
}
else
{
var loadTrans = ImplOptions.LoadTransforms ?? true;
pipe = LoadLoader(rep, ImplOptions.DataFile, loadTrans);
if (loadTrans)
{
trainPipe = pipe;
}
}
if (Utils.Size(ImplOptions.Transforms) > 0)
{
pipe = LegacyCompositeDataLoader.Create(Host, pipe, ImplOptions.Transforms);
}
// Next consider loading the training data's role mapped schema.
trainSchema = null;
if (wantTrainSchema)
{
// First try to get the role mappings.
var trainRoleMappings = ModelFileUtils.LoadRoleMappingsOrNull(Host, rep);
if (trainRoleMappings != null)
{
// Next create the training schema. In the event that the loaded pipeline happens
// to be the training pipe, we can just use that. If it differs, then we need to
// load the full pipeline from the model, relying upon the fact that all loaders
// can be loaded with no data at all, to get their schemas.
if (trainPipe == null)
{
trainPipe = ModelFileUtils.LoadLoader(Host, rep, new MultiFileSource(null), loadTransforms: true);
}
trainSchema = new RoleMappedSchema(trainPipe.Schema, trainRoleMappings);
}
// If the role mappings are null, an alternative would be to fail. However the idea
// is that the scorer should always still succeed, although perhaps with reduced
// functionality, even when the training schema is null, since not all versions of
// TLC models will have the role mappings preserved, I believe. And, we do want to
// maintain backwards compatibility.
}
}
}
protected ILegacyDataLoader LoadLoader(RepositoryReader rep, string path, bool loadTransforms)
{
return(ModelFileUtils.LoadLoader(Host, rep, new MultiFileSource(path), loadTransforms));
}
