/// <summary>
/// Train a model on a single example,
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <param name="trainerMaker"></param>
/// <param name="checker"></param>
private static void TrivialHelper <TOutput>(Func <ITrainerHost, ITrainer <Instances, IPredictorProducing <TOutput> > > trainerMaker, Action <TOutput, TOutput> checker)
{
// The following simple instance should result in a "trivial" predictor for binary classification, regression, and multiclass, I think.
ListInstances instances = new ListInstances();
instances.AddInst(new Float[] { (Float)0.0 }, (Float)0);
instances.CopyMetadata(null);
ITrainerHost host = new TrainHost(new Random(1), 0);
var trainer = trainerMaker(host);
trainer.Train(instances);
IPredictor <Instance, TOutput> predictor = (IPredictor <Instance, TOutput>)trainer.CreatePredictor();
IPredictor <Instance, TOutput> loadedPredictor = default(IPredictor <Instance, TOutput>);
using (Stream stream = new MemoryStream())
{
using (RepositoryWriter writer = RepositoryWriter.CreateNew(stream, false))
{
ModelSaveContext.SaveModel(writer, predictor, "foo");
writer.Commit();
}
stream.Position = 0;
using (RepositoryReader reader = RepositoryReader.Open(stream, false))
{
ModelLoadContext.LoadModel(out loadedPredictor, reader, "foo");
}
Assert.AreNotEqual(default(IPredictor <Instance, TOutput>), loadedPredictor, "did not load expected model");
}
TOutput result = predictor.Predict(instances[0]);
TOutput loadedResult = loadedPredictor.Predict(instances[0]);
checker(result, loadedResult);
}
/// <summary>
/// Attempt to apply the data transform to a different data view source.
/// If the transform in question implements <see cref="ITransformTemplate"/>, <see cref="ITransformTemplate.ApplyToData"/>
/// is called. Otherwise, the transform is serialized into a byte array and then deserialized.
/// </summary>
/// <param name="env">The host to use</param>
/// <param name="transform">The transform to apply.</param>
/// <param name="newSource">The data view to apply the transform to.</param>
/// <returns>The resulting data view.</returns>
public static IDataTransform ApplyTransformToData(IHostEnvironment env, IDataTransform transform, IDataView newSource)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(transform, nameof(transform));
env.CheckValue(newSource, nameof(newSource));
var rebindable = transform as ITransformTemplate;
if (rebindable != null)
{
return(rebindable.ApplyToData(env, newSource));
}
// Revert to serialization.
using (var stream = new MemoryStream())
{
using (var rep = RepositoryWriter.CreateNew(stream, env))
{
ModelSaveContext.SaveModel(rep, transform, "model");
rep.Commit();
}
stream.Position = 0;
using (var rep = RepositoryReader.Open(stream, env))
{
IDataTransform newData;
ModelLoadContext.LoadModel <IDataTransform, SignatureLoadDataTransform>(env,
out newData, rep, "model", newSource);
return(newData);
}
}
}
/// <summary>
/// Saves <paramref name="loader"/> to the specified <paramref name="stream"/>.
/// </summary>
public static void SaveLoader(ILegacyDataLoader loader, Stream stream)
{
Contracts.CheckValue(loader, nameof(loader));
Contracts.CheckValue(stream, nameof(stream));
Contracts.CheckParam(stream.CanWrite, nameof(stream), "Must be writable");
using (var rep = RepositoryWriter.CreateNew(stream))
{
ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel);
rep.Commit();
}
}
public void SaveTo(IHostEnvironment env, Stream outputStream)
{
using (var ch = env.Start("Saving pipeline"))
{
using (var rep = RepositoryWriter.CreateNew(outputStream, ch))
{
ch.Trace("Saving transformer chain");
ModelSaveContext.SaveModel(rep, this, TransformerChain.LoaderSignature);
rep.Commit();
}
}
}
/// <summary>
/// Saves <paramref name="loader"/> to the specified <paramref name="file"/>.
/// </summary>
public static void SaveLoader(IDataLoader loader, IFileHandle file)
{
Contracts.CheckValue(loader, nameof(loader));
Contracts.CheckValue(file, nameof(file));
Contracts.CheckParam(file.CanWrite, nameof(file), "Must be writable");
using (var stream = file.CreateWriteStream())
using (var rep = RepositoryWriter.CreateNew(stream))
{
ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel);
rep.Commit();
}
}
/// <summary>
/// Save this transform model.
/// </summary>
public void Save(IHostEnvironment env, Stream stream)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(stream, nameof(stream));
using (var ch = env.Start("Saving transform model"))
{
using (var rep = RepositoryWriter.CreateNew(stream, ch))
{
ch.Trace("Saving root schema and transformations");
TrainUtils.SaveDataPipe(env, rep, _chain, blankLoader: true);
rep.Commit();
}
}
}
/// <summary>
/// Save a <see cref="ICanForecast{T}"/> model to a file specified by <paramref name="filePath"/>
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="catalog"><see cref="ModelOperationsCatalog"/></param>
/// <param name="model"><see cref="ICanForecast{T}"/> model to save.</param>
/// <param name="filePath">File path to save the model to.</param>
public static void SaveForecastingModel <T>(this ModelOperationsCatalog catalog, ICanForecast <T> model, string filePath)
{
var env = CatalogUtils.GetEnvironment(catalog);
using (var file = File.Create(filePath))
{
using (var ch = env.Start("Saving forecasting model."))
{
using (var rep = RepositoryWriter.CreateNew(file, ch))
{
ModelSaveContext.SaveModel(rep, model, LoaderSignature);
rep.Commit();
}
}
}
}
/// <summary>
/// Save the model to the stream.
/// The method saves the loader and the transformations of dataPipe and saves optionally predictor
/// and command. It also uses featureColumn, if provided, to extract feature names.
/// </summary>
/// <param name="env">The host environment to use.</param>
/// <param name="ch">The communication channel to use.</param>
/// <param name="outputStream">The output model stream.</param>
/// <param name="predictor">The predictor.</param>
/// <param name="data">The training examples.</param>
/// <param name="command">The command string.</param>
public static void SaveModel(IHostEnvironment env, IChannel ch, Stream outputStream, IPredictor predictor, RoleMappedData data, string command = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ch, nameof(ch));
ch.CheckValue(outputStream, nameof(outputStream));
ch.CheckValueOrNull(predictor);
ch.CheckValue(data, nameof(data));
ch.CheckValueOrNull(command);
using (var ch2 = env.Start("SaveModel"))
using (var pch = env.StartProgressChannel("Saving model"))
{
using (var rep = RepositoryWriter.CreateNew(outputStream, ch2))
{
if (predictor != null)
{
ch2.Trace("Saving predictor");
ModelSaveContext.SaveModel(rep, predictor, ModelFileUtils.DirPredictor);
}
ch2.Trace("Saving loader and transformations");
var dataPipe = data.Data;
if (dataPipe is IDataLoader)
{
ModelSaveContext.SaveModel(rep, dataPipe, ModelFileUtils.DirDataLoaderModel);
}
else
{
SaveDataPipe(env, rep, dataPipe);
}
// REVIEW: Handle statistics.
// ModelSaveContext.SaveModel(rep, dataStats, DirDataStats);
if (!string.IsNullOrWhiteSpace(command))
{
using (var ent = rep.CreateEntry(ModelFileUtils.DirTrainingInfo, "Command.txt"))
using (var writer = Utils.OpenWriter(ent.Stream))
writer.WriteLine(command);
}
ModelFileUtils.SaveRoleMappings(env, ch, data.Schema, rep);
rep.Commit();
}
ch2.Done();
}
}
/// <summary>
/// Save the contents to a stream, as a "model file".
/// </summary>
public void SaveTo(IHostEnvironment env, Stream outputStream)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(outputStream, nameof(outputStream));
env.Check(outputStream.CanWrite && outputStream.CanSeek, "Need a writable and seekable stream to save");
using (var ch = env.Start("Saving pipeline"))
{
using (var rep = RepositoryWriter.CreateNew(outputStream, ch))
{
ch.Trace("Saving data reader");
ModelSaveContext.SaveModel(rep, Reader, "Reader");
ch.Trace("Saving transformer chain");
ModelSaveContext.SaveModel(rep, Transformer, TransformerChain.LoaderSignature);
rep.Commit();
}
}
}
public LoaderWrapper(IHostEnvironment env, ModelLoadContext ctx)
{
ctx.CheckAtModel(GetVersionInfo());
ctx.LoadModel <IDataLoader, SignatureLoadDataLoader>(env, out var loader, "Loader", new MultiFileSource(null));
var loaderStream = new MemoryStream();
using (var rep = RepositoryWriter.CreateNew(loaderStream))
{
ModelSaveContext.SaveModel(rep, loader, "Loader");
rep.Commit();
}
_env = env;
_loaderFactory = (IMultiStreamSource source) =>
{
using (var rep = RepositoryReader.Open(loaderStream))
{
ModelLoadContext.LoadModel <IDataLoader, SignatureLoadDataLoader>(env, out var ldr, rep, "Loader", source);
return(ldr);
}
};
}