/// <summary> /// Saves the transformer to file. /// </summary> /// <param name="ctx">The <see cref="ModelSaveContext"/> that facilitates saving to the <see cref="Repository"/>.</param> public void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // model: prediction model. // stream: empty data view that contains train schema. // ids of strings: feature columns. // float: scorer threshold // id of string: scorer threshold column ctx.SaveModel(Model, DirModel); ctx.SaveBinaryStream(DirTransSchema, writer => { using (var ch = Host.Start("Saving train schema")) { var saver = new BinarySaver(Host, new BinarySaver.Arguments { Silent = true }); DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(Host, TrainSchema), writer.BaseStream); } }); for (int i = 0; i < Model.FieldCount; i++) { ctx.SaveString(FeatureColumns[i]); } ctx.Writer.Write(_threshold); ctx.SaveString(_thresholdColumn); }
public sealed override void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); ctx.CheckAtModel(); ctx.SaveModel(Bindable, "SchemaBindableMapper"); SaveCore(ctx); }
public void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // ** Binary format ** // int: tailing directory count // Schema of the loader // int[]: srcColumns // byte[]: subloader // model: file path spec ctx.Writer.Write(_tailingDirCount); // Save the schema var noRows = new EmptyDataView(_host, Schema); var saverArgs = new BinarySaver.Arguments(); saverArgs.Silent = true; var saver = new BinarySaver(_host, saverArgs); using (var strm = new MemoryStream()) { var allColumns = Enumerable.Range(0, Schema.Count).ToArray(); saver.SaveData(strm, noRows, allColumns); ctx.SaveBinaryStream(SchemaCtxName, w => w.WriteByteArray(strm.ToArray())); } ctx.Writer.WriteIntArray(_srcDirIndex); ctx.Writer.WriteByteArray(_subLoaderBytes); ctx.SaveModel(_pathParser, FilePathSpecCtxName); }
public void Save(ModelSaveContext ctx) { _host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var dataPipe = _xf; var transforms = new List <IDataTransform>(); while (dataPipe is IDataTransform xf) { transforms.Add(xf); dataPipe = xf.Source; Contracts.AssertValue(dataPipe); } transforms.Reverse(); ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema)); ctx.Writer.Write(transforms.Count); for (int i = 0; i < transforms.Count; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i], dirName); } }
private static void SaveCore(ModelSaveContext ctx, Action <ModelSaveContext> loaderSaveAction, TransformEx[] transforms) { Contracts.AssertValue(ctx); Contracts.AssertValue(loaderSaveAction); Contracts.AssertValueOrNull(transforms); // *** Binary format *** // int: sizeof(Float) // int: number of transforms // foreach transform: (starting from version VersionAddedTags) // string: tag // string: args string ctx.Writer.Write(sizeof(Float)); ctx.Writer.Write(transforms.Length); using (var loaderCtx = new ModelSaveContext(ctx.Repository, Path.Combine(ctx.Directory ?? "", "Loader"), ModelLoadContext.ModelStreamName)) { loaderSaveAction(loaderCtx); loaderCtx.Done(); } for (int i = 0; i < transforms.Length; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i].Transform, dirName); Contracts.AssertNonEmpty(transforms[i].Tag); ctx.SaveNonEmptyString(transforms[i].Tag); ctx.SaveStringOrNull(transforms[i].ArgsString); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); Host.Assert(InitialWindowSize == 0); Host.Assert(2 <= SeasonalWindowSize); Host.Assert(0 <= DiscountFactor && DiscountFactor <= 1); Host.Assert(Enum.IsDefined(typeof(ErrorFunctionUtils.ErrorFunction), ErrorFunction)); Host.Assert(Model != null); // *** Binary format *** // <base> // int: _seasonalWindowSize // float: _discountFactor // byte: _errorFunction // bool: _isAdaptive // State: StateRef // AdaptiveSingularSpectrumSequenceModeler: _model base.Save(ctx); ctx.Writer.Write(SeasonalWindowSize); ctx.Writer.Write(DiscountFactor); ctx.Writer.Write((byte)ErrorFunction); ctx.Writer.Write(IsAdaptive); StateRef.Save(ctx.Writer); ctx.SaveModel(Model, "SSA"); }
public void Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var dataPipe = _xf; var transforms = new List <IDataTransform>(); while (dataPipe is IDataTransform xf) { // REVIEW: a malicious user could construct a loop in the Source chain, that would // cause this method to iterate forever (and throw something when the list overflows). There's // no way to insulate from ALL malicious behavior. transforms.Add(xf); dataPipe = xf.Source; Contracts.AssertValue(dataPipe); } transforms.Reverse(); ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema)); ctx.Writer.Write(transforms.Count); for (int i = 0; i < transforms.Count; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i], dirName); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // number of columns // for each column, number of slots // Sub-models: // count tables (each in a separate folder) Host.Assert(_countTables.Length > 0); ctx.Writer.Write(_countTables.Length); for (int i = 0; i < _countTables.Length; i++) { var size = _countTables[i].Length; Host.Assert(size > 0); ctx.Writer.Write(size); for (int j = 0; j < size; j++) { var tableName = string.Format("Table_{0:000}_{1:000}", i, j); ctx.SaveModel(_countTables[i][j], tableName); } } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // model: prediction model. // stream: empty data view that contains train schema. // string: feature column name. // string: the name of the columns where tree prediction values are stored. // string: the name of the columns where trees' leave are stored. // string: the name of the columns where trees' paths are stored. ctx.SaveModel(Model, DirModel); ctx.SaveBinaryStream(DirTransSchema, writer => { using (var ch = Host.Start("Saving train schema")) { var saver = new BinarySaver(Host, new BinarySaver.Arguments { Silent = true }); DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(Host, TrainSchema), writer.BaseStream); } }); ctx.SaveString(_featureDetachedColumn.Name); ctx.SaveStringOrNull(_treesColumnName); ctx.SaveStringOrNull(_leavesColumnName); ctx.SaveStringOrNull(_pathsColumnName); }
/// <summary> /// Attempt to apply the data transform to a different data view source. /// If the transform in question implements <see cref="ITransformTemplate"/>, <see cref="ITransformTemplate.ApplyToData"/> /// is called. Otherwise, the transform is serialized into a byte array and then deserialized. /// </summary> /// <param name="env">The host to use</param> /// <param name="transform">The transform to apply.</param> /// <param name="newSource">The data view to apply the transform to.</param> /// <returns>The resulting data view.</returns> public static IDataTransform ApplyTransformToData(IHostEnvironment env, IDataTransform transform, IDataView newSource) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(transform, nameof(transform)); env.CheckValue(newSource, nameof(newSource)); var rebindable = transform as ITransformTemplate; if (rebindable != null) { return(rebindable.ApplyToData(env, newSource)); } // Revert to serialization. using (var stream = new MemoryStream()) { using (var rep = RepositoryWriter.CreateNew(stream, env)) { ModelSaveContext.SaveModel(rep, transform, "model"); rep.Commit(); } stream.Position = 0; using (var rep = RepositoryReader.Open(stream, env)) { IDataTransform newData; ModelLoadContext.LoadModel <IDataTransform, SignatureLoadDataTransform>(env, out newData, rep, "model", newSource); return(newData); } } }
public void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); Contracts.CheckValue(_predictor, "_predictor"); ctx.SaveModel(_predictor, "predictor"); }
protected override void SaveModel(ModelSaveContext ctx) { _host.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _args.Write(ctx, _host); ctx.SaveModel(_scorer, "scorer"); }
public void Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var ldr = Read(new MultiFileSource(null)); ctx.SaveModel(ldr, "Loader"); }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(_trend, "No trend predictor was ever trained. The model cannot be saved."); Host.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _args.Write(ctx, Host); ctx.SaveModel(_trend, "trend"); }
private protected override void SaveCore(ModelSaveContext ctx) { base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of classes ctx.Writer.Write(_numClasses); // Save other streams. for (int i = 0; i < _numClasses; i++) { int index = GetIndex(i, 0); ctx.SaveModel(_predictors[index + i], string.Format(SubPredictorFmt, i)); for (int j = 0; j < i; j++) ctx.SaveModel(_predictors[index + j], string.Format(SubPredictorFmt2, i, j)); } }
public virtual void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); // *** Binary format *** // <nothing> ctx.SaveModel(Predictor, ModelFileUtils.DirPredictor); }
private protected override void SaveModel(ModelSaveContext ctx) { Contracts.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // model: _calibrator ctx.SaveModel(_calibrator, @"Calibrator"); }
protected override void SaveCore(ModelSaveContext ctx) { Host.AssertValue(ctx); // *** Binary format *** // <base> // The combiner ctx.SaveModel(Combiner, "Combiner"); }
public void SaveCore(ModelSaveContext ctx, IHost host, VersionInfo versionInfo) { host.Check(Classes.Count > 0, "The model cannot be saved, it was never trained."); host.Check(Classes.Count == Classes.Length, "The model cannot be saved, it was never trained."); ctx.SetVersionInfo(versionInfo); ctx.Writer.WriteIntArray(Classes.Indices); if (LabelType == NumberDataViewType.Single) { ctx.Writer.WriteSingleArray(Classes.Values as float[]); } else if (LabelType == NumberDataViewType.Byte) { ctx.Writer.WriteByteArray(Classes.Values as byte[]); } else if (LabelType == NumberDataViewType.UInt16) { ctx.Writer.WriteUIntArray((Classes.Values as ushort[]).Select(c => (uint)c).ToArray()); } else if (LabelType == NumberDataViewType.UInt32) { ctx.Writer.WriteUIntArray(Classes.Values as uint[]); } else { throw host.Except("Unexpected type for LabelType."); } ctx.Writer.Write(_singleColumn ? 1 : 0); ctx.Writer.Write(_labelKey ? 1 : 0); var preds = Predictors; ctx.Writer.Write(preds.Length); for (int i = 0; i < preds.Length; i++) { ctx.SaveModel(preds[i], string.Format("M2B{0}", i)); } ctx.Writer.Write(_reclassificationPredictor != null ? (byte)1 : (byte)0); if (_reclassificationPredictor != null) { ctx.SaveModel(_reclassificationPredictor, "Reclassification"); } ctx.Writer.Write((byte)213); }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // _mapper ctx.SaveModel(_mapper, "Mapper"); }
public void Save(ModelSaveContext ctx) { _host.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.Writer.Write(_dataTransforms.Length); for (int i = 0; i < _dataTransforms.Length; ++i) { ctx.SaveModel(_dataTransforms[i], string.Format("XF{0}", i)); } }
/// <summary> /// Saves <paramref name="loader"/> to the specified <paramref name="stream"/>. /// </summary> public static void SaveLoader(ILegacyDataLoader loader, Stream stream) { Contracts.CheckValue(loader, nameof(loader)); Contracts.CheckValue(stream, nameof(stream)); Contracts.CheckParam(stream.CanWrite, nameof(stream), "Must be writable"); using (var rep = RepositoryWriter.CreateNew(stream)) { ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel); rep.Commit(); } }
public void SaveTo(IHostEnvironment env, Stream outputStream) { using (var ch = env.Start("Saving pipeline")) { using (var rep = RepositoryWriter.CreateNew(outputStream, ch)) { ch.Trace("Saving transformer chain"); ModelSaveContext.SaveModel(rep, this, TransformerChain.LoaderSignature); rep.Commit(); } } }
protected virtual void SaveCore(ModelSaveContext ctx) { Host.Assert(Meta != null); // *** Binary format *** // int: sizeof(Single) // Float: _validationDatasetProportion ctx.Writer.Write(sizeof(Single)); ctx.Writer.Write(ValidationDatasetProportion); ctx.SaveModel(Meta, "MetaPredictor"); }
public void Save(ModelSaveContext ctx) { _host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // ensemble _host.AssertValue(_ensemble); ctx.SaveModel(_ensemble, "Ensemble"); }
public override void Save(ModelSaveContext ctx) { _host.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _args.Write(ctx, _host); ctx.Writer.Write(_predictor != null ? (byte)1 : (byte)0); ctx.Writer.Write(_cali != null ? (byte)1 : (byte)0); ctx.Writer.Write(_scorer != null ? (byte)1 : (byte)0); if (_predictor != null) { ctx.SaveModel(_predictor, "predictor"); } if (_cali != null) { ctx.SaveModel(_cali, "calibrator"); } if (_scorer != null) { ctx.SaveModel(_scorer, "scorer"); } }
/// <summary> /// Saves <paramref name="loader"/> to the specified <paramref name="file"/>. /// </summary> public static void SaveLoader(IDataLoader loader, IFileHandle file) { Contracts.CheckValue(loader, nameof(loader)); Contracts.CheckValue(file, nameof(file)); Contracts.CheckParam(file.CanWrite, nameof(file), "Must be writable"); using (var stream = file.CreateWriteStream()) using (var rep = RepositoryWriter.CreateNew(stream)) { ModelSaveContext.SaveModel(rep, loader, ModelFileUtils.DirDataLoaderModel); rep.Commit(); } }
protected void Write(ModelSaveContext ctx) { _args.Write(ctx, _host); ctx.Writer.Write((byte)177); if (_args.serialize) { if (_predictor == null) { throw _host.Except("_predictor cannot be null."); } ctx.SaveModel(_predictor, "predictor"); } }
private protected override void SaveCore(ModelSaveContext ctx) { base.SaveCore(ctx); // *** Binary format *** // int: model count // int: weight count (0 or model count) // Single[]: weights // for each model: // int: number of SelectedFeatures (in bits) // byte[]: selected features (as many as needed for number of bits == (numSelectedFeatures + 7) / 8) // int: number of Metric values // for each Metric: // Single: metric value // int: metric name (id of the metric name in the string table) ctx.Writer.Write(Models.Length); ctx.Writer.WriteSingleArray(Weights); // Save other streams. for (int i = 0; i < Models.Length; i++) { var model = Models[i]; ctx.SaveModel(model.Predictor, string.Format(SubPredictorFmt, i)); Host.AssertValueOrNull(model.SelectedFeatures); ctx.Writer.WriteBitArray(model.SelectedFeatures); Host.AssertValueOrNull(model.Metrics); int numMetrics = Utils.Size(model.Metrics); ctx.Writer.Write(numMetrics); for (int j = 0; j < numMetrics; j++) { var metric = model.Metrics[j]; ctx.Writer.Write((Single)metric.Value); ctx.SaveStringOrNull(metric.Key); } } ctx.SaveModel(Combiner, @"Combiner"); }
public void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // byte[]: A chunk of data saving both the type and value of the label names, as saved by the BinarySaver. // int: string id of the metadata kind ctx.SaveModel(_bindable, _innerDir); Utils.MarshalActionInvoke(SaveCore <int>, _type.ItemType.RawType, ctx); ctx.SaveNonEmptyString(_metadataKind); }