public static void Save(ModelSaveContext ctx, ref VBuffer <DvText> names)
{
Contracts.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: number of features (size)
// int: number of indices (-1 if dense)
// int[]: indices (if sparse)
// int[]: ids of names (matches either number of features or number of indices)
ctx.Writer.Write(names.Length);
if (names.IsDense)
{
ctx.Writer.Write(-1);
for (int i = 0; i < names.Length; i++)
{
ctx.SaveStringOrNull(names.Values[i].ToString());
}
}
else
{
ctx.Writer.Write(names.Count);
for (int ii = 0; ii < names.Count; ii++)
{
ctx.Writer.Write(names.Indices[ii]);
}
for (int ii = 0; ii < names.Count; ii++)
{
ctx.SaveStringOrNull(names.Values[ii].ToString());
}
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// model: prediction model.
// stream: empty data view that contains train schema.
// string: feature column name.
// string: the name of the columns where tree prediction values are stored.
// string: the name of the columns where trees' leave are stored.
// string: the name of the columns where trees' paths are stored.
ctx.SaveModel(Model, DirModel);
ctx.SaveBinaryStream(DirTransSchema, writer =>
{
using (var ch = Host.Start("Saving train schema"))
{
var saver = new BinarySaver(Host, new BinarySaver.Arguments {
Silent = true
});
DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(Host, TrainSchema), writer.BaseStream);
}
});
ctx.SaveString(_featureDetachedColumn.Name);
ctx.SaveStringOrNull(_treesColumnName);
ctx.SaveStringOrNull(_leavesColumnName);
ctx.SaveStringOrNull(_pathsColumnName);
}
private protected override void SaveCore(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// <base info>
// id of string: train label column
base.SaveCore(ctx);
ctx.SaveStringOrNull(_trainLabelColumn);
ctx.SaveStringOrNull(_scoreColumn);
ctx.SaveStringOrNull(_predictedLabelColumn);
}
private static void SaveCore(ModelSaveContext ctx, Action <ModelSaveContext> loaderSaveAction, TransformEx[] transforms)
{
Contracts.AssertValue(ctx);
Contracts.AssertValue(loaderSaveAction);
Contracts.AssertValueOrNull(transforms);
// *** Binary format ***
// int: sizeof(Float)
// int: number of transforms
// foreach transform: (starting from version VersionAddedTags)
// string: tag
// string: args string
ctx.Writer.Write(sizeof(Float));
ctx.Writer.Write(transforms.Length);
using (var loaderCtx = new ModelSaveContext(ctx.Repository, Path.Combine(ctx.Directory ?? "", "Loader"), ModelLoadContext.ModelStreamName))
{
loaderSaveAction(loaderCtx);
loaderCtx.Done();
}
for (int i = 0; i < transforms.Length; i++)
{
var dirName = string.Format(TransformDirTemplate, i);
ctx.SaveModel(transforms[i].Transform, dirName);
Contracts.AssertNonEmpty(transforms[i].Tag);
ctx.SaveNonEmptyString(transforms[i].Tag);
ctx.SaveStringOrNull(transforms[i].ArgsString);
}
}
public virtual void Save(ModelSaveContext ctx)
{
// *** Binary format **
// int: Id of the score column name
// int: Id of the label column name
ctx.SaveNonEmptyString(ScoreCol);
ctx.SaveStringOrNull(LabelCol);
}
public void Save(ModelSaveContext ctx)
{
// *** Binary format ***
// int: the stopwords list language
// int: the id of languages column name
ctx.Writer.Write((int)Lang);
Contracts.Assert((LangsColIndex >= 0 && _langsColName != null) ||
(LangsColIndex == -1 && _langsColName == null));
ctx.SaveStringOrNull(_langsColName);
}
internal void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of columns
// foreach column:
// int: id of column name
// byte: DataKind
// byte: bool of whether this is a key type
// for a key type:
// ulong: count for key range
// int: number of segments
// foreach segment:
// string id: name
// int: min
// int: lim
// byte: force vector (verWrittenCur: verIsVectorSupported)
ctx.Writer.Write(Infos.Length);
for (int iinfo = 0; iinfo < Infos.Length; iinfo++)
{
var info = Infos[iinfo];
ctx.SaveNonEmptyString(info.Name);
var type = info.ColType.GetItemType();
InternalDataKind rawKind = type.GetRawKind();
Contracts.Assert((InternalDataKind)(byte)rawKind == rawKind);
ctx.Writer.Write((byte)rawKind);
ctx.Writer.WriteBoolByte(type is KeyDataViewType);
if (type is KeyDataViewType key)
{
ctx.Writer.Write(key.Count);
}
if (info.Segments is null)
{
ctx.Writer.Write(0);
}
else
{
ctx.Writer.Write(info.Segments.Length);
foreach (var seg in info.Segments)
{
ctx.SaveStringOrNull(seg.Name);
ctx.Writer.Write(seg.Min);
ctx.Writer.Write(seg.Lim);
ctx.Writer.WriteBoolByte(seg.ForceVector);
}
}
}
}
public void Save(ModelSaveContext ctx)
{
// *** Binary format ***
// int: Number of trees
// Regression trees (num trees of these)
// double: Bias
// int: Id to InputInitializationContent string
BinaryWriter writer = ctx.Writer;
writer.Write(NumTrees);
foreach (RegressionTree tree in Trees)
{
tree.Save(ctx);
}
writer.Write(Bias);
ctx.SaveStringOrNull(_firstInputInitializationContent);
}
protected virtual void SaveCore(ModelSaveContext ctx)
{
// *** Binary format ***
// model: prediction model.
// stream: empty data view that contains train schema.
// id of string: feature column.
ctx.SaveModel(Model, DirModel);
ctx.SaveBinaryStream(DirTransSchema, writer =>
{
using (var ch = Host.Start("Saving train schema"))
{
var saver = new BinarySaver(Host, new BinarySaver.Arguments {
Silent = true
});
DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(Host, TrainSchema), writer.BaseStream);
}
});
ctx.SaveStringOrNull(FeatureColumn);
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: count of group column infos (ie, count of source columns)
// For each group column info
// int: the tokenizer language
// int: the id of source column name
// int: the id of languages column name
// bool: whether the types output is required
// For each column info that belongs to this group column info
// (either one column info for tokens or two for tokens and types)
// int: the id of the column name
Contracts.Assert(Utils.Size(GroupInfos) > 0);
ctx.Writer.Write(GroupInfos.Length);
int iinfo = 0;
for (int groupId = 0; groupId < GroupInfos.Length; groupId++)
{
var groupInfo = GroupInfos[groupId];
Contracts.Assert(Enum.IsDefined(typeof(Language), groupInfo.Lang));
ctx.Writer.Write((int)groupInfo.Lang);
ctx.SaveNonEmptyString(groupInfo.SrcColName);
ctx.SaveStringOrNull(groupInfo.LangsColName);
ctx.Writer.WriteBoolByte(groupInfo.RequireTypes);
int count = groupInfo.RequireTypes ? 2 : 1;
int lim = iinfo + count;
for (; iinfo < lim; iinfo++)
{
Contracts.Assert(Infos[iinfo].GroupInfoId == groupId);
ctx.SaveNonEmptyString(GetColumnNameCore(iinfo));
}
}
Contracts.Assert(iinfo == Infos.Length);
}
private protected override void SaveCore(ModelSaveContext ctx)
{
base.SaveCore(ctx);
// *** Binary format ***
// int: model count
// int: weight count (0 or model count)
// Single[]: weights
// for each model:
// int: number of SelectedFeatures (in bits)
// byte[]: selected features (as many as needed for number of bits == (numSelectedFeatures + 7) / 8)
// int: number of Metric values
// for each Metric:
// Single: metric value
// int: metric name (id of the metric name in the string table)
ctx.Writer.Write(Models.Length);
ctx.Writer.WriteSingleArray(Weights);
// Save other streams.
for (int i = 0; i < Models.Length; i++)
{
var model = Models[i];
ctx.SaveModel(model.Predictor, string.Format(SubPredictorFmt, i));
Host.AssertValueOrNull(model.SelectedFeatures);
ctx.Writer.WriteBitArray(model.SelectedFeatures);
Host.AssertValueOrNull(model.Metrics);
int numMetrics = Utils.Size(model.Metrics);
ctx.Writer.Write(numMetrics);
for (int j = 0; j < numMetrics; j++)
{
var metric = model.Metrics[j];
ctx.Writer.Write((Single)metric.Value);
ctx.SaveStringOrNull(metric.Key);
}
}
ctx.SaveModel(Combiner, @"Combiner");
}
private protected virtual void SaveCore(ModelSaveContext ctx)
{
SaveModelCore(ctx);
ctx.SaveStringOrNull(FeatureColumnName);
}