void ICanSaveModel.Save(ModelSaveContext ctx)
{
if (!_allowSave)
{
throw _host.Except("Saving is not permitted.");
}
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
var dataPipe = _xf;
var transforms = new List <IDataTransform>();
while (dataPipe is IDataTransform xf)
{
// REVIEW: a malicious user could construct a loop in the Source chain, that would
// cause this method to iterate forever (and throw something when the list overflows). There's
// no way to insulate from ALL malicious behavior.
transforms.Add(xf);
dataPipe = xf.Source;
Contracts.AssertValue(dataPipe);
}
transforms.Reverse();
ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema));
ctx.Writer.Write(transforms.Count);
for (int i = 0; i < transforms.Count; i++)
{
var dirName = string.Format(TransformDirTemplate, i);
ctx.SaveModel(transforms[i], dirName);
}
}
public void Save(ModelSaveContext ctx)
{
_host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
var dataPipe = _xf;
var transforms = new List <IDataTransform>();
while (dataPipe is IDataTransform xf)
{
transforms.Add(xf);
dataPipe = xf.Source;
Contracts.AssertValue(dataPipe);
}
transforms.Reverse();
ctx.SaveSubModel("Loader", c => BinaryLoader.SaveInstance(_host, c, dataPipe.Schema));
ctx.Writer.Write(transforms.Count);
for (int i = 0; i < transforms.Count; i++)
{
var dirName = string.Format(TransformDirTemplate, i);
ctx.SaveModel(transforms[i], dirName);
}
}
public static void SaveAll(IHost host, ModelSaveContext ctx, int infoLim, VBuffer <ReadOnlyMemory <char> >[] keyValues)
{
Contracts.AssertValue(host);
host.AssertValue(ctx);
host.AssertValueOrNull(keyValues);
if (keyValues == null)
{
return;
}
using (var ch = host.Start("SaveTextValues"))
{
// Save the key names as separate submodels.
const string dirFormat = "Vocabulary_{0:000}";
CodecFactory factory = new CodecFactory(host);
for (int iinfo = 0; iinfo < infoLim; iinfo++)
{
if (keyValues[iinfo].Length == 0)
{
continue;
}
ctx.SaveSubModel(string.Format(dirFormat, iinfo),
c => Save(ch, c, factory, ref keyValues[iinfo]));
}
ch.Done();
}
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// for each term map:
// bool(byte): whether this column should present key value metadata as text
SaveBase(ctx);
Host.Assert(_termMap.Length == Infos.Length);
Host.Assert(_textMetadata.Length == Infos.Length);
ctx.Writer.WriteBoolBytesNoCount(_textMetadata, _textMetadata.Length);
// REVIEW: Should we do separate sub models for each dictionary?
const string dir = "Vocabulary";
ctx.SaveSubModel(dir,
c =>
{
// *** Binary format ***
// int: number of term maps (should equal number of columns)
// for each term map:
// byte: code identifying the term map type (0 text, 1 codec)
// <data>: type specific format, see TermMap save/load methods
Host.CheckValue(c, nameof(ctx));
c.CheckAtModel();
c.SetVersionInfo(GetTermManagerVersionInfo());
c.Writer.Write(_termMap.Length);
foreach (var term in _termMap)
{
term.Map.Save(c, this);
}
c.SaveTextStream("Terms.txt",
writer =>
{
foreach (var map in _termMap)
{
map.WriteTextTerms(writer);
}
});
});
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// <base>
SaveBase(ctx);
const string dir = "Stopwords";
ctx.SaveSubModel(dir,
c =>
{
Host.CheckValue(c, nameof(ctx));
c.CheckAtModel();
c.SetVersionInfo(GetStopwrodsManagerVersionInfo());
// *** Binary format ***
// int: number of stopwords
// int[]: stopwords string ids
Host.Assert(_stopWordsMap.Count > 0);
ctx.Writer.Write(_stopWordsMap.Count);
int id = 0;
foreach (var nstr in _stopWordsMap)
{
Host.Assert(nstr.Id == id);
ctx.SaveString(nstr.Value);
id++;
}
ctx.SaveTextStream("Stopwords.txt", writer =>
{
foreach (var nstr in _stopWordsMap)
{
writer.WriteLine("{0}\t{1}", nstr.Id, nstr.Value);
}
});
});
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// <base>
ctx.Writer.Write(sizeof(Float));
SaveBase(ctx);
// Individual normalization models.
Host.Assert(_functions.Length == Infos.Length);
for (int iinfo = 0; iinfo < _functions.Length; iinfo++)
{
Host.Assert(Infos[iinfo].TypeSrc.ValueCount > 0);
var dir = string.Format("Normalizer_{0:000}", iinfo);
Host.Assert(_functions[iinfo] != null);
ctx.SaveSubModel(dir, _functions[iinfo].Save);
}
}