void ICanSaveModel.Save(ModelSaveContext ctx) { if (!_allowSave) { throw _host.Except("Saving is not permitted."); } ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var dataPipe = _xf; var transforms = new List <IDataTransform>(); while (dataPipe is IDataTransform xf) { // REVIEW: a malicious user could construct a loop in the Source chain, that would // cause this method to iterate forever (and throw something when the list overflows). There's // no way to insulate from ALL malicious behavior. transforms.Add(xf); dataPipe = xf.Source; Contracts.AssertValue(dataPipe); } transforms.Reverse(); ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema)); ctx.Writer.Write(transforms.Count); for (int i = 0; i < transforms.Count; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i], dirName); } }
public void Save(ModelSaveContext ctx) { _host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var dataPipe = _xf; var transforms = new List <IDataTransform>(); while (dataPipe is IDataTransform xf) { transforms.Add(xf); dataPipe = xf.Source; Contracts.AssertValue(dataPipe); } transforms.Reverse(); ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema)); ctx.Writer.Write(transforms.Count); for (int i = 0; i < transforms.Count; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i], dirName); } }
public static void SaveAll(IHost host, ModelSaveContext ctx, int infoLim, VBuffer <ReadOnlyMemory <char> >[] keyValues) { Contracts.AssertValue(host); host.AssertValue(ctx); host.AssertValueOrNull(keyValues); if (keyValues == null) { return; } using (var ch = host.Start("SaveTextValues")) { // Save the key names as separate submodels. const string dirFormat = "Vocabulary_{0:000}"; CodecFactory factory = new CodecFactory(host); for (int iinfo = 0; iinfo < infoLim; iinfo++) { if (keyValues[iinfo].Length == 0) { continue; } ctx.SaveSubModel(string.Format(dirFormat, iinfo), c => Save(ch, c, factory, ref keyValues[iinfo])); } ch.Done(); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // for each term map: // bool(byte): whether this column should present key value metadata as text SaveBase(ctx); Host.Assert(_termMap.Length == Infos.Length); Host.Assert(_textMetadata.Length == Infos.Length); ctx.Writer.WriteBoolBytesNoCount(_textMetadata, _textMetadata.Length); // REVIEW: Should we do separate sub models for each dictionary? const string dir = "Vocabulary"; ctx.SaveSubModel(dir, c => { // *** Binary format *** // int: number of term maps (should equal number of columns) // for each term map: // byte: code identifying the term map type (0 text, 1 codec) // <data>: type specific format, see TermMap save/load methods Host.CheckValue(c, nameof(ctx)); c.CheckAtModel(); c.SetVersionInfo(GetTermManagerVersionInfo()); c.Writer.Write(_termMap.Length); foreach (var term in _termMap) { term.Map.Save(c, this); } c.SaveTextStream("Terms.txt", writer => { foreach (var map in _termMap) { map.WriteTextTerms(writer); } }); }); }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // <base> SaveBase(ctx); const string dir = "Stopwords"; ctx.SaveSubModel(dir, c => { Host.CheckValue(c, nameof(ctx)); c.CheckAtModel(); c.SetVersionInfo(GetStopwrodsManagerVersionInfo()); // *** Binary format *** // int: number of stopwords // int[]: stopwords string ids Host.Assert(_stopWordsMap.Count > 0); ctx.Writer.Write(_stopWordsMap.Count); int id = 0; foreach (var nstr in _stopWordsMap) { Host.Assert(nstr.Id == id); ctx.SaveString(nstr.Value); id++; } ctx.SaveTextStream("Stopwords.txt", writer => { foreach (var nstr in _stopWordsMap) { writer.WriteLine("{0}\t{1}", nstr.Id, nstr.Value); } }); }); }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // <base> ctx.Writer.Write(sizeof(Float)); SaveBase(ctx); // Individual normalization models. Host.Assert(_functions.Length == Infos.Length); for (int iinfo = 0; iinfo < _functions.Length; iinfo++) { Host.Assert(Infos[iinfo].TypeSrc.ValueCount > 0); var dir = string.Format("Normalizer_{0:000}", iinfo); Host.Assert(_functions[iinfo] != null); ctx.SaveSubModel(dir, _functions[iinfo].Save); } }