public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// <base>
// ldaState[num infos]: The LDA parameters
ctx.Writer.Write(sizeof(Float));
SaveBase(ctx);
Host.Assert(_ldas.Length == Infos.Length);
VBuffer <ReadOnlyMemory <char> > slotNames = default;
for (int i = 0; i < _ldas.Length; i++)
{
GetSlotNames(i, ref slotNames);
_ldas[i].Save(ctx, _saveText, slotNames);
}
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// <base>
// for each added column
// byte: data kind, with high bit set if there is a key range
// if there is a key range
// ulong: min
// int: count (0 for unspecified)
// byte: contiguous
ctx.Writer.Write(sizeof(Float));
SaveBase(ctx);
for (int i = 0; i < _exes.Length; i++)
{
var ex = _exes[i];
Host.Assert((DataKind)(byte)ex.Kind == ex.Kind);
if (!ex.HasKeyRange)
{
ctx.Writer.Write((byte)ex.Kind);
}
else
{
Host.Assert(ex.TypeDst.ItemType.IsKey);
var key = ex.TypeDst.ItemType.AsKey;
byte b = (byte)ex.Kind;
b |= 0x80;
ctx.Writer.Write(b);
ctx.Writer.Write(key.Min);
ctx.Writer.Write(key.Count);
ctx.Writer.WriteBoolByte(key.Contiguous);
}
}
}
public void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Check(CanSave(), "Cannot save this transform as it was not specified as being savable");
ctx.CheckAtModel();
ctx.SetVersionInfo(SerializableLambdaTransform.GetVersionInfo());
// *** Binary format ***
// int: Number of bytes the load method was serialized to
// byte[n]: The serialized load method info
// <arbitrary>: Arbitrary bytes saved by the save action
Host.AssertNonEmpty(_loadFuncBytes);
ctx.Writer.WriteByteArray(_loadFuncBytes);
using (var ms = new MemoryStream())
{
using (var writer = new BinaryWriter(ms, Encoding.UTF8, leaveOpen: true))
_saveAction(writer);
ctx.Writer.WriteByteArray(ms.ToArray());
}
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// <base>
ctx.Writer.Write(sizeof(Float));
SaveBase(ctx);
// Individual normalization models.
Host.Assert(_functions.Length == Infos.Length);
for (int iinfo = 0; iinfo < _functions.Length; iinfo++)
{
Host.Assert(Infos[iinfo].TypeSrc.ValueCount > 0);
var dir = string.Format("Normalizer_{0:000}", iinfo);
Host.Assert(_functions[iinfo] != null);
ctx.SaveSubModel(dir, _functions[iinfo].Save);
}
}
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: number of rows for the sample pool
// bool(as byte): whether randomize only this transform, and not its input
// bool(as byte): whether present a shuffled cursor even when a shuffled cursor is not requested
// bool(as byte): whether the input cursor is always shuffled even when a shuffled cursor is not requested
// int, only present if one of the previous two were true: seed for the random number generator to use
// when a shuffled cursor was not requested
ctx.Writer.Write(_poolRows);
ctx.Writer.WriteBoolByte(_poolOnly);
ctx.Writer.WriteBoolByte(_forceShuffle);
ctx.Writer.WriteBoolByte(_forceShuffleSource);
if (_forceShuffle || _forceShuffleSource)
{
ctx.Writer.Write(_forceShuffleSeed);
}
}
protected override void SaveModel(ModelSaveContext ctx)
{
_host.CheckValue(ctx, "ctx");
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
_args.Write(ctx, _host);
ctx.Writer.Write(_predictor != null ? (byte)1 : (byte)0);
ctx.Writer.Write(_cali != null ? (byte)1 : (byte)0);
ctx.Writer.Write(_scorer != null ? (byte)1 : (byte)0);
if (_predictor != null)
{
ctx.SaveModel(_predictor, "predictor");
}
if (_cali != null)
{
ctx.SaveModel(_cali, "calibrator");
}
if (_scorer != null)
{
ctx.SaveModel(_scorer, "scorer");
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: number of output columns
// for each output column:
// int: number of inputs
// foreach input
// int: Id of the input column name
// int: Id of the expression
// int: Id of the output column name
// int: the index of the vector input (or -1)
// int[]: The data kinds of the input columns
ctx.Writer.Write(_columns.Length);
for (int i = 0; i < _columns.Length; i++)
{
var inputColumnNames = _columns[i].InputColumnNames;
ctx.Writer.Write(inputColumnNames.Length);
for (int j = 0; j < inputColumnNames.Length; j++)
{
ctx.SaveNonEmptyString(inputColumnNames[j]);
}
ctx.SaveNonEmptyString(_columns[i].Expression);
ctx.SaveNonEmptyString(_columns[i].OutputColumnName);
Host.Assert(_columns[i].VectorInputColumn >= -1);
ctx.Writer.Write(_columns[i].VectorInputColumn);
for (int j = 0; j < _columns[i].InputKinds.Length; j++)
{
ctx.Writer.Write(_columns[i].InputKinds[j].ToIndex());
}
}
}
internal void SaveThis(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
Host.Assert(InitialWindowSize == 0);
Host.Assert(Model != null);
// *** Binary format ***
// <base>
// bool: _isAdaptive
// int32: Horizon
// State: StateRef
// AdaptiveSingularSpectrumSequenceModeler: _model
base.SaveModel(ctx);
ctx.Writer.Write(IsAdaptive);
ctx.Writer.Write(Horizon);
ctx.Writer.Write(ConfidenceLevel);
StateRef.Save(ctx.Writer);
ctx.SaveModel(Model, "SSA");
}
public override void Save(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); });
Host.CheckNonEmpty(Inputs, nameof(Inputs));
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
Host.CheckNonEmpty(Outputs, nameof(Outputs));
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: sizeof(Float)
// int: id of column name
// double: min
// double: max
// byte: complement
// byte: includeMin
// byte: includeMax
ctx.Writer.Write(sizeof(float));
ctx.SaveNonEmptyString(Source.Schema[_index].Name);
Host.Assert(_min < _max);
ctx.Writer.Write(_min);
ctx.Writer.Write(_max);
ctx.Writer.WriteBoolByte(_complement);
ctx.Writer.WriteBoolByte(_includeMin);
ctx.Writer.WriteBoolByte(_includeMax);
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
var buffer = new TFBuffer();
_session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
Contracts.AssertNonEmpty(_inputColNames);
ctx.Writer.Write(_inputColNames.Length);
foreach (var colName in _inputColNames)
{
ctx.SaveNonEmptyString(colName);
}
ctx.SaveNonEmptyString(_outputColName);
}
/// <summary>
/// Save model to the given context
/// </summary>
public void Save(ModelSaveContext ctx)
{
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// int: number of rows (m), the limit on row
// int: number of columns (n), the limit on column
// int: rank of factor matrices (k)
// float[m * k]: the left factor matrix
// float[k * n]: the right factor matrix
_host.Check(_numberOfRows > 0, "Number of rows must be positive");
_host.Check(_numberofColumns > 0, "Number of columns must be positive");
_host.Check(_approximationRank > 0, "Number of latent factors must be positive");
ctx.Writer.Write(_numberOfRows);
ctx.Writer.Write(_numberofColumns);
ctx.Writer.Write(_approximationRank);
_host.Check(Utils.Size(_leftFactorMatrix) == _numberOfRows * _approximationRank, "Unexpected matrix size of a factor matrix (matrix P in LIBMF paper)");
_host.Check(Utils.Size(_rightFactorMatrix) == _numberofColumns * _approximationRank, "Unexpected matrix size of a factor matrix (matrix Q in LIBMF paper)");
Utils.WriteSinglesNoCount(ctx.Writer, _leftFactorMatrix.AsSpan(0, _numberOfRows * _approximationRank));
Utils.WriteSinglesNoCount(ctx.Writer, _rightFactorMatrix.AsSpan(0, _numberofColumns * _approximationRank));
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
var buffer = new TFBuffer();
Session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
_host.AssertNonEmpty(Inputs);
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
_host.AssertNonEmpty(Outputs);
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelStream.Name)); });
Host.CheckNonEmpty(Inputs, nameof(Inputs));
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
Host.CheckNonEmpty(Outputs, nameof(Outputs));
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
// Save custom-provided shapes. Those shapes overwrite shapes loaded from the ONNX model file.
int customShapeInfosLength = _options.CustomShapeInfos != null ? _options.CustomShapeInfos.Length : 0;
ctx.Writer.Write(customShapeInfosLength);
for (int i = 0; i < customShapeInfosLength; ++i)
{
var info = _options.CustomShapeInfos[i];
ctx.SaveNonEmptyString(info.Name);
ctx.Writer.WriteIntArray(info.Shape);
}
ctx.Writer.Write(_options.RecursionLimit);
}
private static void Save(IChannel ch, ModelSaveContext ctx, CodecFactory factory, ref VBuffer <ReadOnlyMemory <char> > values)
{
Contracts.AssertValue(ch);
ch.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// Codec parameterization: A codec parameterization that should be a ReadOnlyMemory codec
// int: n, the number of bytes used to write the values
// byte[n]: As encoded using the codec
// Get the codec from the factory
IValueCodec codec;
var result = factory.TryGetCodec(new VectorType(TextType.Instance), out codec);
ch.Assert(result);
ch.Assert(codec.Type.IsVector);
ch.Assert(codec.Type.VectorSize == 0);
ch.Assert(codec.Type.ItemType.RawType == typeof(ReadOnlyMemory <char>));
IValueCodec <VBuffer <ReadOnlyMemory <char> > > textCodec = (IValueCodec <VBuffer <ReadOnlyMemory <char> > >)codec;
factory.WriteCodec(ctx.Writer.BaseStream, codec);
using (var mem = new MemoryStream())
{
using (var writer = textCodec.OpenWriter(mem))
{
writer.Write(ref values);
writer.Commit();
}
ctx.Writer.WriteByteArray(mem.ToArray());
}
// Make this resemble, more or less, the auxiliary output from the TermTransform.
// It will differ somewhat due to the vector being possibly sparse. To distinguish
// between missing and empty, empties are not written at all, while missings are.
var v = values;
char[] buffer = null;
ctx.SaveTextStream("Terms.txt",
writer =>
{
writer.WriteLine("# Number of terms = {0} of length {1}", v.Count, v.Length);
foreach (var pair in v.Items())
{
var text = pair.Value;
if (text.IsEmpty)
{
continue;
}
writer.Write("{0}\t", pair.Key);
// REVIEW: What about escaping this, *especially* for linebreaks?
// Do C# and .NET really have no equivalent to Python's "repr"? :(
if (text.IsEmpty)
{
writer.WriteLine();
continue;
}
Utils.EnsureSize(ref buffer, text.Length);
var span = text.Span;
for (int i = 0; i < text.Length; i++)
{
buffer[i] = span[i];
}
writer.WriteLine(buffer, 0, text.Length);
}
});
}
private protected sealed override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// byte: indicator for frozen models
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
var isFrozen = string.IsNullOrEmpty(_savedModelPath);
ctx.Writer.WriteBoolByte(isFrozen);
if (isFrozen)
{
var buffer = new TFBuffer();
Session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
}
else
{
ctx.SaveBinaryStream("TFSavedModel", w =>
{
string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories);
w.Write(modelFilePaths.Length);
foreach (var fullPath in modelFilePaths)
{
var relativePath = fullPath.Substring(_savedModelPath.Length + 1);
w.Write(relativePath);
using (var fs = new FileStream(fullPath, FileMode.Open))
{
long fileLength = fs.Length;
w.Write(fileLength);
long actualWritten = fs.CopyRange(w.BaseStream, fileLength);
_host.Assert(actualWritten == fileLength);
}
}
});
}
_host.AssertNonEmpty(Inputs);
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
_host.AssertNonEmpty(Outputs);
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
public void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}
void ICanSaveModel.Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}
public void Save(ModelSaveContext ctx)
{
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
_transform.Save(ctx);
}
void ICanSaveModel.Save(ModelSaveContext ctx)
{
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
((ICanSaveModel)_transform).Save(ctx);
}
private protected virtual void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}