public void Write(ModelSaveContext ctx, IHost host)
{
ctx.Writer.Write(Column1x1.ArrayToLine(columns));
ctx.Writer.Write(degree);
ctx.Writer.Write(numThreads ?? -1);
}
void ICanSaveModel.Save(ModelSaveContext ctx) => _parent.SaveModel(ctx);
private protected override void SaveModel(ModelSaveContext ctx) => _parent.SaveModel(ctx);
public void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}
private protected override void SaveModel(ModelSaveContext ctx)
{
(_parent as ICanSaveModel).Save(ctx);
}
private protected override void SaveModel(ModelSaveContext ctx)
{
Contracts.CheckValue(ctx, nameof(ctx));
ctx.SetVersionInfo(GetVersionInfo());
base.SaveModel(ctx);
}
private protected virtual void SaveModel(ModelSaveContext ctx) => InternalTransform.SaveThis(ctx);
public void Save(ModelSaveContext ctx)
{
}
public void Save(ModelSaveContext ctx)
{
_host.AssertValue(ctx);
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
// *** Binary format ***
// byte: indicator for frozen models
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
var isFrozen = string.IsNullOrEmpty(_savedModelPath);
ctx.Writer.WriteBoolByte(isFrozen);
if (isFrozen)
{
var buffer = new TFBuffer();
Session.Graph.ToGraphDef(buffer);
ctx.SaveBinaryStream("TFModel", w =>
{
w.WriteByteArray(buffer.ToArray());
});
}
else
{
ctx.SaveBinaryStream("TFSavedModel", w =>
{
string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories);
w.Write(modelFilePaths.Length);
foreach (var fullPath in modelFilePaths)
{
var relativePath = fullPath.Substring(_savedModelPath.Length + 1);
w.Write(relativePath);
using (var fs = new FileStream(fullPath, FileMode.Open))
{
long fileLength = fs.Length;
w.Write(fileLength);
long actualWritten = fs.CopyRange(w.BaseStream, fileLength);
_host.Assert(actualWritten == fileLength);
}
}
});
}
_host.AssertNonEmpty(Inputs);
ctx.Writer.Write(Inputs.Length);
foreach (var colName in Inputs)
{
ctx.SaveNonEmptyString(colName);
}
_host.AssertNonEmpty(Outputs);
ctx.Writer.Write(Outputs.Length);
foreach (var colName in Outputs)
{
ctx.SaveNonEmptyString(colName);
}
}
public static void Save(ModelSaveContext ctx, in VBuffer <ReadOnlyMemory <char> > names)
internal abstract void SaveModel(ModelSaveContext ctx);
void ICanSaveModel.Save(ModelSaveContext ctx) => _transform.SaveModel(ctx);
protected override void SaveCore(ModelSaveContext ctx)
{
Host.AssertValue(ctx);
Bindings.Save(ctx);
}
protected override void SaveCore(ModelSaveContext ctx)
{
base.SaveCore(ctx);
ctx.Writer.Write((byte)_impl.LabelType.RawKind());
_impl.SaveCore(ctx, Host, GetVersionInfo());
}
public override void Save(ModelSaveContext ctx)
{
_parent.Save(ctx);
}
protected abstract void SaveCore(ModelSaveContext ctx);
protected override void SaveCore(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
ctx.SetVersionInfo(GetVersionInfo());
_bindings.Save(ctx);
}
/// <summary>
/// For saving a model into a repository.
/// </summary>
public virtual void Save(ModelSaveContext ctx)
{
InternalTransform.SaveThis(ctx);
}
void ICanSaveModel.Save(ModelSaveContext ctx) => (_transformer as ICanSaveModel).Save(ctx);
private protected abstract void SaveModel(ModelSaveContext ctx);
protected override void SaveCore(ModelSaveContext ctx)
{
base.SaveCore(ctx);
ctx.SetVersionInfo(GetVersionInfo());
Host.Assert(_biases.Length == _numClasses);
Host.Assert(_biases.Length == _weights.Length);
#if DEBUG
foreach (var fw in _weights)
{
Host.Assert(fw.Length == _numFeatures);
}
#endif
// *** Binary format ***
// int: number of features
// int: number of classes = number of biases
// float[]: biases
// (weight matrix, in CSR if sparse)
// (see https://netlib.org/linalg/html_templates/node91.html#SECTION00931100000000000000)
// int: number of row start indices (_numClasses + 1 if sparse, 0 if dense)
// int[]: row start indices
// int: total number of column indices (0 if dense)
// int[]: column index of each non-zero weight
// int: total number of non-zero weights (same as number of column indices if sparse, num of classes * num of features if dense)
// float[]: non-zero weights
// bool: whether label names are present
// int[]: Id of label names (optional, in a separate stream)
// LinearModelStatistics: model statistics (optional, in a separate stream)
ctx.Writer.Write(_numFeatures);
ctx.Writer.Write(_numClasses);
ctx.Writer.WriteFloatsNoCount(_biases, _numClasses);
// _weights == _weighsDense means we checked that all vectors in _weights
// are actually dense, and so we assigned the same object, or it came dense
// from deserialization.
if (_weights == _weightsDense)
{
ctx.Writer.Write(0); // Number of starts.
ctx.Writer.Write(0); // Number of indices.
ctx.Writer.Write(_numFeatures * _weights.Length);
foreach (var fv in _weights)
{
Host.Assert(fv.Length == _numFeatures);
ctx.Writer.WriteFloatsNoCount(fv.Values, _numFeatures);
}
}
else
{
// Number of starts.
ctx.Writer.Write(_numClasses + 1);
// Starts always starts with 0.
int numIndices = 0;
ctx.Writer.Write(numIndices);
for (int i = 0; i < _weights.Length; i++)
{
// REVIEW: Assuming the presence of *any* zero justifies
// writing in sparse format seems stupid, but might be difficult
// to change without changing the format since the presence of
// any sparse vector means we're writing indices anyway. Revisit.
// This is actually a bug waiting to happen: sparse/dense vectors
// can have different dot products even if they are logically the
// same vector.
numIndices += NonZeroCount(ref _weights[i]);
ctx.Writer.Write(numIndices);
}
ctx.Writer.Write(numIndices);
{
// just scoping the count so we can use another further down
int count = 0;
foreach (var fw in _weights)
{
if (fw.IsDense)
{
for (int i = 0; i < fw.Length; i++)
{
if (fw.Values[i] != 0)
{
ctx.Writer.Write(i);
count++;
}
}
}
else
{
ctx.Writer.WriteIntsNoCount(fw.Indices, fw.Count);
count += fw.Count;
}
}
Host.Assert(count == numIndices);
}
ctx.Writer.Write(numIndices);
{
int count = 0;
foreach (var fw in _weights)
{
if (fw.IsDense)
{
for (int i = 0; i < fw.Length; i++)
{
if (fw.Values[i] != 0)
{
ctx.Writer.Write(fw.Values[i]);
count++;
}
}
}
else
{
ctx.Writer.WriteFloatsNoCount(fw.Values, fw.Count);
count += fw.Count;
}
}
Host.Assert(count == numIndices);
}
}
Contracts.AssertValueOrNull(_labelNames);
if (_labelNames != null)
{
ctx.SaveBinaryStream(LabelNamesSubModelFilename, w => SaveLabelNames(ctx, w));
}
Contracts.AssertValueOrNull(_stats);
if (_stats != null)
{
using (var statsCtx = new ModelSaveContext(ctx.Repository,
Path.Combine(ctx.Directory ?? "", ModelStatsSubModelFilename), ModelLoadContext.ModelStreamName))
{
_stats.Save(statsCtx);
statsCtx.Done();
}
}
}
private protected sealed override void SaveModel(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
SaveCore(ctx);
}
protected virtual void SaveCore(ModelSaveContext ctx)
{
SaveModel(ctx);
ctx.SaveStringOrNull(FeatureColumn);
}
public void Save(ModelSaveContext ctx)
{
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
_transform.Save(ctx);
}
public void Save(ModelSaveContext ctx) => _transformer.Save(ctx);
public override void Save(ModelSaveContext ctx)
{
Contracts.CheckValue(ctx, nameof(ctx));
ctx.SetVersionInfo(GetVersionInfo());
base.Save(ctx);
}
void ICanSaveModel.Save(ModelSaveContext ctx) => SaveModel(ctx);
public void Save(ModelSaveContext ctx)
{
_parent.Save(ctx);
}
private protected override void SaveCore(ModelSaveContext ctx)
{
base.SaveCore(ctx);
ctx.SetVersionInfo(GetVersionInfo());
}
public void Save(ModelSaveContext ctx)
{
// Needed by the API but does nothing.
}
