public override void Save(ModelSaveContext ctx)
{
// *** Binary format ***
// (base fields)
// if version >= VersionAddedAliases
// foreach column:
// foreach non-null alias
// int: index of the alias
// int: string id of the alias
// int: -1, marks the end of the list
base.Save(ctx);
Contracts.Assert(_aliases.Length == Infos.Length);
for (int i = 0; i < Infos.Length; i++)
{
Contracts.Assert(_aliases[i].Length == Infos[i].SrcIndices.Length);
for (int j = 0; j < _aliases[i].Length; j++)
{
if (!string.IsNullOrEmpty(_aliases[i][j]))
{
ctx.Writer.Write(j);
ctx.SaveNonEmptyString(_aliases[i][j]);
}
}
ctx.Writer.Write(-1);
}
}
private static void SaveCore(ModelSaveContext ctx, Action <ModelSaveContext> loaderSaveAction, TransformEx[] transforms)
{
Contracts.AssertValue(ctx);
Contracts.AssertValue(loaderSaveAction);
Contracts.AssertValueOrNull(transforms);
// *** Binary format ***
// int: sizeof(Float)
// int: number of transforms
// foreach transform: (starting from version VersionAddedTags)
// string: tag
// string: args string
ctx.Writer.Write(sizeof(Float));
ctx.Writer.Write(transforms.Length);
using (var loaderCtx = new ModelSaveContext(ctx.Repository, Path.Combine(ctx.Directory ?? "", "Loader"), ModelLoadContext.ModelStreamName))
{
loaderSaveAction(loaderCtx);
loaderCtx.Done();
}
for (int i = 0; i < transforms.Length; i++)
{
var dirName = string.Format(TransformDirTemplate, i);
ctx.SaveModel(transforms[i].Transform, dirName);
Contracts.AssertNonEmpty(transforms[i].Tag);
ctx.SaveNonEmptyString(transforms[i].Tag);
ctx.SaveStringOrNull(transforms[i].ArgsString);
}
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of added columns
// for each added column
// int: id of output column name
// byte: useCounter
// if !useCounter
// uint: seed0
// uint: seed1
// uint: seed2
// uint: seed3
int size = InfoCount;
ctx.Writer.Write(size);
for (int i = 0; i < size; i++)
{
ctx.SaveNonEmptyString(GetColumnNameCore(i));
ctx.Writer.WriteBoolByte(UseCounter[i]);
if (!UseCounter[i])
{
States[i].Save(ctx.Writer);
}
}
}
internal void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of added columns
// for each added column
// int: id of output column name
// int: id of input column name
ctx.Writer.Write(Infos.Length);
foreach (var info in Infos)
{
ctx.SaveNonEmptyString(info.Name);
ctx.SaveNonEmptyString(Input[info.Source].Name);
}
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: id of output column name
ctx.SaveNonEmptyString(GetColumnNameCore(0));
}
public virtual void Save(ModelSaveContext ctx)
{
// *** Binary format **
// int: Id of the score column name
// int: Id of the label column name
ctx.SaveNonEmptyString(ScoreCol);
ctx.SaveStringOrNull(LabelCol);
}
public override void Save(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); });
Host.CheckNonEmpty(Inputs, nameof(Inputs));
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
ctx.SaveNonEmptyString(colName);
Host.CheckNonEmpty(Outputs, nameof(Outputs));
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
ctx.SaveNonEmptyString(colName);
}
protected void SaveBase(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: id of the suffix
// int: the number of input column roles
// for each input column:
// int: id of the column role
// int: id of the column name
ctx.SaveString(Suffix);
var cols = Mapper.GetInputColumnRoles().ToArray();
ctx.Writer.Write(cols.Length);
foreach (var kvp in cols)
{
ctx.SaveNonEmptyString(kvp.Key.Value);
ctx.SaveNonEmptyString(kvp.Value);
}
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: count of group column infos (ie, count of source columns)
// For each group column info
// int: the tokenizer language
// int: the id of source column name
// int: the id of languages column name
// bool: whether the types output is required
// For each column info that belongs to this group column info
// (either one column info for tokens or two for tokens and types)
// int: the id of the column name
Contracts.Assert(Utils.Size(GroupInfos) > 0);
ctx.Writer.Write(GroupInfos.Length);
int iinfo = 0;
for (int groupId = 0; groupId < GroupInfos.Length; groupId++)
{
var groupInfo = GroupInfos[groupId];
Contracts.Assert(Enum.IsDefined(typeof(Language), groupInfo.Lang));
ctx.Writer.Write((int)groupInfo.Lang);
ctx.SaveNonEmptyString(groupInfo.SrcColName);
ctx.SaveStringOrNull(groupInfo.LangsColName);
ctx.Writer.WriteBoolByte(groupInfo.RequireTypes);
int count = groupInfo.RequireTypes ? 2 : 1;
int lim = iinfo + count;
for (; iinfo < lim; iinfo++)
{
Contracts.Assert(Infos[iinfo].GroupInfoId == groupId);
ctx.SaveNonEmptyString(GetColumnNameCore(iinfo));
}
}
Contracts.Assert(iinfo == Infos.Length);
}
public virtual void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Assert(InitialWindowSize >= 0);
Host.Assert(WindowSize >= 0);
// *** Binary format ***
// int: _windowSize
// int: _initialWindowSize
// int (string ID): _inputColumnName
// int (string ID): _outputColumnName
// ColumnType: _transform.Schema.GetColumnType(0)
ctx.Writer.Write(WindowSize);
ctx.Writer.Write(InitialWindowSize);
ctx.SaveNonEmptyString(InputColumnName);
ctx.SaveNonEmptyString(OutputColumnName);
var bs = new BinarySaver(Host, new BinarySaver.Arguments());
bs.TryWriteTypeDescription(ctx.Writer.BaseStream, OutputColumnType, out int byteWritten);
}
public virtual void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of added columns
// for each added column
// int: id of output column name
// int: number of input column names
// int[]: ids of input column names
ctx.Writer.Write(Infos.Length);
for (int i = 0; i < Infos.Length; i++)
{
var info = Infos[i];
ctx.SaveNonEmptyString(GetColumnNameCore(i));
ctx.Writer.Write(info.SrcIndices.Length);
foreach (int src in info.SrcIndices)
{
ctx.SaveNonEmptyString(Input.GetColumnName(src));
}
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: number of output columns
// for each output column:
// int: number of inputs
// foreach input
// int: Id of the input column name
// int: Id of the expression
// int: Id of the output column name
// int: the index of the vector input (or -1)
// int[]: The data kinds of the input columns
ctx.Writer.Write(_columns.Length);
for (int i = 0; i < _columns.Length; i++)
{
var inputColumnNames = _columns[i].InputColumnNames;
ctx.Writer.Write(inputColumnNames.Length);
for (int j = 0; j < inputColumnNames.Length; j++)
{
ctx.SaveNonEmptyString(inputColumnNames[j]);
}
ctx.SaveNonEmptyString(_columns[i].Expression);
ctx.SaveNonEmptyString(_columns[i].OutputColumnName);
Host.Assert(_columns[i].VectorInputColumn >= -1);
ctx.Writer.Write(_columns[i].VectorInputColumn);
for (int j = 0; j < _columns[i].InputKinds.Length; j++)
{
ctx.Writer.Write(_columns[i].InputKinds[j].ToIndex());
}
}
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
var buffer = new TFBuffer();
_session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
Contracts.AssertNonEmpty(_inputColNames);
ctx.Writer.Write(_inputColNames.Length);
foreach (var colName in _inputColNames)
{
ctx.SaveNonEmptyString(colName);
}
ctx.SaveNonEmptyString(_outputColName);
}
public void Save(ModelSaveContext ctx)
{
_ectx.AssertValue(ctx);
// *** Binary format ***
// int: ungroup mode
// int: K - number of pivot columns
// int[K]: ids of pivot column names
ctx.Writer.Write((int)Mode);
ctx.Writer.Write(_infos.Length);
foreach (var ex in _infos)
ctx.SaveNonEmptyString(ex.Name);
}
internal void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of columns
// foreach column:
// int: id of column name
// byte: DataKind
// byte: bool of whether this is a key type
// for a key type:
// ulong: count for key range
// int: number of segments
// foreach segment:
// string id: name
// int: min
// int: lim
// byte: force vector (verWrittenCur: verIsVectorSupported)
ctx.Writer.Write(Infos.Length);
for (int iinfo = 0; iinfo < Infos.Length; iinfo++)
{
var info = Infos[iinfo];
ctx.SaveNonEmptyString(info.Name);
var type = info.ColType.GetItemType();
InternalDataKind rawKind = type.GetRawKind();
Contracts.Assert((InternalDataKind)(byte)rawKind == rawKind);
ctx.Writer.Write((byte)rawKind);
ctx.Writer.WriteBoolByte(type is KeyDataViewType);
if (type is KeyDataViewType key)
{
ctx.Writer.Write(key.Count);
}
if (info.Segments is null)
{
ctx.Writer.Write(0);
}
else
{
ctx.Writer.Write(info.Segments.Length);
foreach (var seg in info.Segments)
{
ctx.SaveStringOrNull(seg.Name);
ctx.Writer.Write(seg.Min);
ctx.Writer.Write(seg.Lim);
ctx.Writer.WriteBoolByte(seg.ForceVector);
}
}
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelFile)); });
Host.CheckNonEmpty(Inputs, nameof(Inputs));
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
Host.CheckNonEmpty(Outputs, nameof(Outputs));
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// byte: default HiddenColumnOption value
// int: number of raw column infos
// for each raw column info
// int: id of output column name
// int: id of input column name
// byte: HiddenColumnOption
Contracts.Assert((HiddenColumnOption)(byte)HidDefault == HidDefault);
ctx.Writer.Write((byte)HidDefault);
ctx.Writer.Write(RawInfos.Length);
for (int i = 0; i < RawInfos.Length; i++)
{
var raw = RawInfos[i];
ctx.SaveNonEmptyString(raw.Name);
ctx.SaveNonEmptyString(raw.Source);
Contracts.Assert((HiddenColumnOption)(byte)raw.Hid == raw.Hid);
ctx.Writer.Write((byte)raw.Hid);
}
}
public void Save(ModelSaveContext ctx)
{
Contracts.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// byte[]: A chunk of data saving both the type and value of the label names, as saved by the BinarySaver.
// int: string id of the metadata kind
ctx.SaveModel(_bindable, _innerDir);
Utils.MarshalActionInvoke(SaveCore <int>, _type.ItemType.RawType, ctx);
ctx.SaveNonEmptyString(_metadataKind);
}
private protected virtual void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Assert(InitialWindowSize >= 0);
Host.Assert(WindowSize >= 0);
// *** Binary format ***
// int: _windowSize
// int: _initialWindowSize
// int (string ID): _sourceColumnName
// int (string ID): _outputColumnName
// ColumnType: _transform.Schema.GetColumnType(0)
ctx.Writer.Write(WindowSize);
ctx.Writer.Write(InitialWindowSize);
ctx.SaveNonEmptyString(InputColumnName);
ctx.SaveNonEmptyString(OutputColumnName);
ctx.Writer.Write(ForecastingConfidenceIntervalMinOutputColumnName ?? string.Empty);
ctx.Writer.Write(ForecastingConfidenceIntervalMaxOutputColumnName ?? string.Empty);
var bs = new BinarySaver(Host, new BinarySaver.Arguments());
bs.TryWriteTypeDescription(ctx.Writer.BaseStream, OutputColumnType, out int byteWritten);
}
public void Save(ModelSaveContext ctx)
{
ctx.SetVersionInfo(GetVersionInfo());
ctx.Writer.WriteBoolByte(KeepColumns);
ctx.Writer.WriteBoolByte(KeepHidden);
ctx.Writer.WriteBoolByte(IgnoreMissing);
var length = _selectedColumns.Length;
ctx.Writer.Write(length);
for (int i = 0; i < length; i++)
{
ctx.SaveNonEmptyString(_selectedColumns[i]);
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelStream.Name)); });
Host.CheckNonEmpty(Inputs, nameof(Inputs));
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
Host.CheckNonEmpty(Outputs, nameof(Outputs));
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
// Save custom-provided shapes. Those shapes overwrite shapes loaded from the ONNX model file.
int customShapeInfosLength = _options.CustomShapeInfos != null ? _options.CustomShapeInfos.Length : 0;
ctx.Writer.Write(customShapeInfosLength);
for (int i = 0; i < customShapeInfosLength; ++i)
{
var info = _options.CustomShapeInfos[i];
ctx.SaveNonEmptyString(info.Name);
ctx.Writer.WriteIntArray(info.Shape);
}
ctx.Writer.Write(_options.RecursionLimit);
}
public void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// bool: whether to keep (vs drop) the named columns
// int: number of names
// int[]: the ids of the names
ctx.Writer.WriteBoolByte(Keep);
ctx.Writer.Write(Names.Count);
foreach (var name in Names)
{
ctx.SaveNonEmptyString(name);
}
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
var buffer = new TFBuffer();
Session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
_host.AssertNonEmpty(Inputs);
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
_host.AssertNonEmpty(Outputs);
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// int: number of columns
// int[]: ids of column names
ctx.Writer.Write(sizeof(Float));
Host.Assert(_infos.Length > 0);
ctx.Writer.Write(_infos.Length);
foreach (var info in _infos)
{
ctx.SaveNonEmptyString(Source.Schema.GetColumnName(info.Index));
}
}
public override void Save(ModelSaveContext ctx)
{
Contracts.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format **
// base
// int: _scoreSize
// int[]: Ids of the quantile names
base.Save(ctx);
Host.Assert(_scoreSize > 0);
ctx.Writer.Write(_scoreSize);
for (int i = 0; i < _scoreSize; i++)
{
ctx.SaveNonEmptyString(_quantiles[i].ToString());
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// int: id of column name
// double: min
// double: max
// byte: complement
// byte: includeMin
// byte: includeMax
ctx.Writer.Write(sizeof(float));
ctx.SaveNonEmptyString(Source.Schema[_index].Name);
Host.Assert(_min < _max);
ctx.Writer.Write(_min);
ctx.Writer.Write(_max);
ctx.Writer.WriteBoolByte(_complement);
ctx.Writer.WriteBoolByte(_includeMin);
ctx.Writer.WriteBoolByte(_includeMax);
}
internal void Save(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// int: number of columns
// foreach column:
// int: id of column name
// byte: DataKind
// byte: bool of whether this is a key type
// for a key type:
// ulong: count for key range
// byte: bool of whether the source index is valid
// for a valid source index:
// int: source index
ctx.Writer.Write(Infos.Length);
for (int iinfo = 0; iinfo < Infos.Length; iinfo++)
{
var info = Infos[iinfo];
ctx.SaveNonEmptyString(info.Name);
var type = info.ColType.GetItemType();
InternalDataKind rawKind = type.GetRawKind();
Contracts.Assert((InternalDataKind)(byte)rawKind == rawKind);
ctx.Writer.Write((byte)rawKind);
ctx.Writer.WriteBoolByte(type is KeyDataViewType);
if (type is KeyDataViewType key)
{
ctx.Writer.Write(key.Count);
}
ctx.Writer.WriteBoolByte(info.SourceIndex.HasValue);
if (info.SourceIndex.HasValue)
{
ctx.Writer.Write(info.SourceIndex.GetValueOrDefault());
}
}
}
public void Save(IHostEnvironment env, ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
// *** Binary format ***
// Schema of the data view containing the optional columns
// int: number of added columns
// for each added column
// int: id of output column name
// ColumnType: the type of the column
var noRows = new EmptyDataView(env, _inputWithOptionalColumn);
var saverArgs = new BinarySaver.Arguments();
saverArgs.Silent = true;
var saver = new BinarySaver(env, saverArgs);
using (var strm = new MemoryStream())
{
saver.SaveData(strm, noRows, _srcColsWithOptionalColumn);
ctx.SaveBinaryStream("Schema.idv", w => w.WriteByteArray(strm.ToArray()));
}
int size = InfoCount;
ctx.Writer.Write(size);
saver = new BinarySaver(env, new BinarySaver.Arguments());
for (int i = 0; i < size; i++)
{
ctx.SaveNonEmptyString(GetColumnNameCore(i));
var columnType = ColumnTypes[i];
int written;
saver.TryWriteTypeDescription(ctx.Writer.BaseStream, columnType, out written);
}
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// byte: indicator for frozen models
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
var isFrozen = string.IsNullOrEmpty(_savedModelPath);
ctx.Writer.WriteBoolByte(isFrozen);
if (isFrozen)
{
var buffer = new TFBuffer();
Session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
}
else
{
ctx.SaveBinaryStream("TFSavedModel", w =>
{
string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories);
w.Write(modelFilePaths.Length);
foreach (var fullPath in modelFilePaths)
{
var relativePath = fullPath.Substring(_savedModelPath.Length + 1);
w.Write(relativePath);
using (var fs = new FileStream(fullPath, FileMode.Open))
{
long fileLength = fs.Length;
w.Write(fileLength);
long actualWritten = fs.CopyRange(w.BaseStream, fileLength);
_host.Assert(actualWritten == fileLength);
}
}
});
}
_host.AssertNonEmpty(Inputs);
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
_host.AssertNonEmpty(Outputs);
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}