/// <summary> /// Train a model on a single example, /// </summary> /// <typeparam name="TOutput"></typeparam> /// <param name="trainerMaker"></param> /// <param name="checker"></param> private static void TrivialHelper <TOutput>(Func <ITrainerHost, ITrainer <Instances, IPredictorProducing <TOutput> > > trainerMaker, Action <TOutput, TOutput> checker) { // The following simple instance should result in a "trivial" predictor for binary classification, regression, and multiclass, I think. ListInstances instances = new ListInstances(); instances.AddInst(new Float[] { (Float)0.0 }, (Float)0); instances.CopyMetadata(null); ITrainerHost host = new TrainHost(new Random(1), 0); var trainer = trainerMaker(host); trainer.Train(instances); IPredictor <Instance, TOutput> predictor = (IPredictor <Instance, TOutput>)trainer.CreatePredictor(); IPredictor <Instance, TOutput> loadedPredictor = default(IPredictor <Instance, TOutput>); using (Stream stream = new MemoryStream()) { using (RepositoryWriter writer = RepositoryWriter.CreateNew(stream, false)) { ModelSaveContext.SaveModel(writer, predictor, "foo"); writer.Commit(); } stream.Position = 0; using (RepositoryReader reader = RepositoryReader.Open(stream, false)) { ModelLoadContext.LoadModel(out loadedPredictor, reader, "foo"); } Assert.AreNotEqual(default(IPredictor <Instance, TOutput>), loadedPredictor, "did not load expected model"); } TOutput result = predictor.Predict(instances[0]); TOutput loadedResult = loadedPredictor.Predict(instances[0]); checker(result, loadedResult); }
/// <summary> /// Attempt to apply the data transform to a different data view source. /// If the transform in question implements <see cref="ITransformTemplate"/>, <see cref="ITransformTemplate.ApplyToData"/> /// is called. Otherwise, the transform is serialized into a byte array and then deserialized. /// </summary> /// <param name="env">The host to use</param> /// <param name="transform">The transform to apply.</param> /// <param name="newSource">The data view to apply the transform to.</param> /// <returns>The resulting data view.</returns> public static IDataTransform ApplyTransformToData(IHostEnvironment env, IDataTransform transform, IDataView newSource) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(transform, nameof(transform)); env.CheckValue(newSource, nameof(newSource)); var rebindable = transform as ITransformTemplate; if (rebindable != null) { return(rebindable.ApplyToData(env, newSource)); } // Revert to serialization. using (var stream = new MemoryStream()) { using (var rep = RepositoryWriter.CreateNew(stream, env)) { ModelSaveContext.SaveModel(rep, transform, "model"); rep.Commit(); } stream.Position = 0; using (var rep = RepositoryReader.Open(stream, env)) { IDataTransform newData; ModelLoadContext.LoadModel <IDataTransform, SignatureLoadDataTransform>(env, out newData, rep, "model", newSource); return(newData); } } }
/// <summary> /// Saves <paramref name="loader"/> to the specified <paramref name="stream"/>. /// </summary> public static void SaveLoader(ILegacyDataLoader loader, Stream stream) { Contracts.CheckValue(loader, nameof(loader)); Contracts.CheckValue(stream, nameof(stream)); Contracts.CheckParam(stream.CanWrite, nameof(stream), "Must be writable"); using (var rep = RepositoryWriter.CreateNew(stream)) { ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel); rep.Commit(); } }
public void SaveTo(IHostEnvironment env, Stream outputStream) { using (var ch = env.Start("Saving pipeline")) { using (var rep = RepositoryWriter.CreateNew(outputStream, ch)) { ch.Trace("Saving transformer chain"); ModelSaveContext.SaveModel(rep, this, TransformerChain.LoaderSignature); rep.Commit(); } } }
/// <summary> /// Saves <paramref name="loader"/> to the specified <paramref name="file"/>. /// </summary> public static void SaveLoader(IDataLoader loader, IFileHandle file) { Contracts.CheckValue(loader, nameof(loader)); Contracts.CheckValue(file, nameof(file)); Contracts.CheckParam(file.CanWrite, nameof(file), "Must be writable"); using (var stream = file.CreateWriteStream()) using (var rep = RepositoryWriter.CreateNew(stream)) { ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel); rep.Commit(); } }
/// <summary> /// Save this transform model. /// </summary> public void Save(IHostEnvironment env, Stream stream) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(stream, nameof(stream)); using (var ch = env.Start("Saving transform model")) { using (var rep = RepositoryWriter.CreateNew(stream, ch)) { ch.Trace("Saving root schema and transformations"); TrainUtils.SaveDataPipe(env, rep, _chain, blankLoader: true); rep.Commit(); } } }
/// <summary> /// Save a <see cref="ICanForecast{T}"/> model to a file specified by <paramref name="filePath"/> /// </summary> /// <typeparam name="T"></typeparam> /// <param name="catalog"><see cref="ModelOperationsCatalog"/></param> /// <param name="model"><see cref="ICanForecast{T}"/> model to save.</param> /// <param name="filePath">File path to save the model to.</param> public static void SaveForecastingModel <T>(this ModelOperationsCatalog catalog, ICanForecast <T> model, string filePath) { var env = CatalogUtils.GetEnvironment(catalog); using (var file = File.Create(filePath)) { using (var ch = env.Start("Saving forecasting model.")) { using (var rep = RepositoryWriter.CreateNew(file, ch)) { ModelSaveContext.SaveModel(rep, model, LoaderSignature); rep.Commit(); } } } }
/// <summary> /// Save the model to the stream. /// The method saves the loader and the transformations of dataPipe and saves optionally predictor /// and command. It also uses featureColumn, if provided, to extract feature names. /// </summary> /// <param name="env">The host environment to use.</param> /// <param name="ch">The communication channel to use.</param> /// <param name="outputStream">The output model stream.</param> /// <param name="predictor">The predictor.</param> /// <param name="data">The training examples.</param> /// <param name="command">The command string.</param> public static void SaveModel(IHostEnvironment env, IChannel ch, Stream outputStream, IPredictor predictor, RoleMappedData data, string command = null) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(ch, nameof(ch)); ch.CheckValue(outputStream, nameof(outputStream)); ch.CheckValueOrNull(predictor); ch.CheckValue(data, nameof(data)); ch.CheckValueOrNull(command); using (var ch2 = env.Start("SaveModel")) using (var pch = env.StartProgressChannel("Saving model")) { using (var rep = RepositoryWriter.CreateNew(outputStream, ch2)) { if (predictor != null) { ch2.Trace("Saving predictor"); ModelSaveContext.SaveModel(rep, predictor, ModelFileUtils.DirPredictor); } ch2.Trace("Saving loader and transformations"); var dataPipe = data.Data; if (dataPipe is IDataLoader) { ModelSaveContext.SaveModel(rep, dataPipe, ModelFileUtils.DirDataLoaderModel); } else { SaveDataPipe(env, rep, dataPipe); } // REVIEW: Handle statistics. // ModelSaveContext.SaveModel(rep, dataStats, DirDataStats); if (!string.IsNullOrWhiteSpace(command)) { using (var ent = rep.CreateEntry(ModelFileUtils.DirTrainingInfo, "Command.txt")) using (var writer = Utils.OpenWriter(ent.Stream)) writer.WriteLine(command); } ModelFileUtils.SaveRoleMappings(env, ch, data.Schema, rep); rep.Commit(); } ch2.Done(); } }
/// <summary> /// Save the contents to a stream, as a "model file". /// </summary> public void SaveTo(IHostEnvironment env, Stream outputStream) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(outputStream, nameof(outputStream)); env.Check(outputStream.CanWrite && outputStream.CanSeek, "Need a writable and seekable stream to save"); using (var ch = env.Start("Saving pipeline")) { using (var rep = RepositoryWriter.CreateNew(outputStream, ch)) { ch.Trace("Saving data reader"); ModelSaveContext.SaveModel(rep, Reader, "Reader"); ch.Trace("Saving transformer chain"); ModelSaveContext.SaveModel(rep, Transformer, TransformerChain.LoaderSignature); rep.Commit(); } } }
public LoaderWrapper(IHostEnvironment env, ModelLoadContext ctx) { ctx.CheckAtModel(GetVersionInfo()); ctx.LoadModel <IDataLoader, SignatureLoadDataLoader>(env, out var loader, "Loader", new MultiFileSource(null)); var loaderStream = new MemoryStream(); using (var rep = RepositoryWriter.CreateNew(loaderStream)) { ModelSaveContext.SaveModel(rep, loader, "Loader"); rep.Commit(); } _env = env; _loaderFactory = (IMultiStreamSource source) => { using (var rep = RepositoryReader.Open(loaderStream)) { ModelLoadContext.LoadModel <IDataLoader, SignatureLoadDataLoader>(env, out var ldr, rep, "Loader", source); return(ldr); } }; }