public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // <base> // ldaState[num infos]: The LDA parameters ctx.Writer.Write(sizeof(Float)); SaveBase(ctx); Host.Assert(_ldas.Length == Infos.Length); VBuffer <ReadOnlyMemory <char> > slotNames = default; for (int i = 0; i < _ldas.Length; i++) { GetSlotNames(i, ref slotNames); _ldas[i].Save(ctx, _saveText, slotNames); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // <base> // for each added column // byte: data kind, with high bit set if there is a key range // if there is a key range // ulong: min // int: count (0 for unspecified) // byte: contiguous ctx.Writer.Write(sizeof(Float)); SaveBase(ctx); for (int i = 0; i < _exes.Length; i++) { var ex = _exes[i]; Host.Assert((DataKind)(byte)ex.Kind == ex.Kind); if (!ex.HasKeyRange) { ctx.Writer.Write((byte)ex.Kind); } else { Host.Assert(ex.TypeDst.ItemType.IsKey); var key = ex.TypeDst.ItemType.AsKey; byte b = (byte)ex.Kind; b |= 0x80; ctx.Writer.Write(b); ctx.Writer.Write(key.Min); ctx.Writer.Write(key.Count); ctx.Writer.WriteBoolByte(key.Contiguous); } } }
public void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); Host.Check(CanSave(), "Cannot save this transform as it was not specified as being savable"); ctx.CheckAtModel(); ctx.SetVersionInfo(SerializableLambdaTransform.GetVersionInfo()); // *** Binary format *** // int: Number of bytes the load method was serialized to // byte[n]: The serialized load method info // <arbitrary>: Arbitrary bytes saved by the save action Host.AssertNonEmpty(_loadFuncBytes); ctx.Writer.WriteByteArray(_loadFuncBytes); using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms, Encoding.UTF8, leaveOpen: true)) _saveAction(writer); ctx.Writer.WriteByteArray(ms.ToArray()); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // <base> ctx.Writer.Write(sizeof(Float)); SaveBase(ctx); // Individual normalization models. Host.Assert(_functions.Length == Infos.Length); for (int iinfo = 0; iinfo < _functions.Length; iinfo++) { Host.Assert(Infos[iinfo].TypeSrc.ValueCount > 0); var dir = string.Format("Normalizer_{0:000}", iinfo); Host.Assert(_functions[iinfo] != null); ctx.SaveSubModel(dir, _functions[iinfo].Save); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of rows for the sample pool // bool(as byte): whether randomize only this transform, and not its input // bool(as byte): whether present a shuffled cursor even when a shuffled cursor is not requested // bool(as byte): whether the input cursor is always shuffled even when a shuffled cursor is not requested // int, only present if one of the previous two were true: seed for the random number generator to use // when a shuffled cursor was not requested ctx.Writer.Write(_poolRows); ctx.Writer.WriteBoolByte(_poolOnly); ctx.Writer.WriteBoolByte(_forceShuffle); ctx.Writer.WriteBoolByte(_forceShuffleSource); if (_forceShuffle || _forceShuffleSource) { ctx.Writer.Write(_forceShuffleSeed); } }
protected override void SaveModel(ModelSaveContext ctx) { _host.CheckValue(ctx, "ctx"); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _args.Write(ctx, _host); ctx.Writer.Write(_predictor != null ? (byte)1 : (byte)0); ctx.Writer.Write(_cali != null ? (byte)1 : (byte)0); ctx.Writer.Write(_scorer != null ? (byte)1 : (byte)0); if (_predictor != null) { ctx.SaveModel(_predictor, "predictor"); } if (_cali != null) { ctx.SaveModel(_cali, "calibrator"); } if (_scorer != null) { ctx.SaveModel(_scorer, "scorer"); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of output columns // for each output column: // int: number of inputs // foreach input // int: Id of the input column name // int: Id of the expression // int: Id of the output column name // int: the index of the vector input (or -1) // int[]: The data kinds of the input columns ctx.Writer.Write(_columns.Length); for (int i = 0; i < _columns.Length; i++) { var inputColumnNames = _columns[i].InputColumnNames; ctx.Writer.Write(inputColumnNames.Length); for (int j = 0; j < inputColumnNames.Length; j++) { ctx.SaveNonEmptyString(inputColumnNames[j]); } ctx.SaveNonEmptyString(_columns[i].Expression); ctx.SaveNonEmptyString(_columns[i].OutputColumnName); Host.Assert(_columns[i].VectorInputColumn >= -1); ctx.Writer.Write(_columns[i].VectorInputColumn); for (int j = 0; j < _columns[i].InputKinds.Length; j++) { ctx.Writer.Write(_columns[i].InputKinds[j].ToIndex()); } } }
internal void SaveThis(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); Host.Assert(InitialWindowSize == 0); Host.Assert(Model != null); // *** Binary format *** // <base> // bool: _isAdaptive // int32: Horizon // State: StateRef // AdaptiveSingularSpectrumSequenceModeler: _model base.SaveModel(ctx); ctx.Writer.Write(IsAdaptive); ctx.Writer.Write(Horizon); ctx.Writer.Write(ConfidenceLevel); StateRef.Save(ctx.Writer); ctx.SaveModel(Model, "SSA"); }
public override void Save(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // int: id of column name // double: min // double: max // byte: complement // byte: includeMin // byte: includeMax ctx.Writer.Write(sizeof(float)); ctx.SaveNonEmptyString(Source.Schema[_index].Name); Host.Assert(_min < _max); ctx.Writer.Write(_min); ctx.Writer.Write(_max); ctx.Writer.WriteBoolByte(_complement); ctx.Writer.WriteBoolByte(_includeMin); ctx.Writer.WriteBoolByte(_includeMax); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var buffer = new TFBuffer(); _session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); Contracts.AssertNonEmpty(_inputColNames); ctx.Writer.Write(_inputColNames.Length); foreach (var colName in _inputColNames) { ctx.SaveNonEmptyString(colName); } ctx.SaveNonEmptyString(_outputColName); }
/// <summary> /// Save model to the given context /// </summary> public void Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of rows (m), the limit on row // int: number of columns (n), the limit on column // int: rank of factor matrices (k) // float[m * k]: the left factor matrix // float[k * n]: the right factor matrix _host.Check(_numberOfRows > 0, "Number of rows must be positive"); _host.Check(_numberofColumns > 0, "Number of columns must be positive"); _host.Check(_approximationRank > 0, "Number of latent factors must be positive"); ctx.Writer.Write(_numberOfRows); ctx.Writer.Write(_numberofColumns); ctx.Writer.Write(_approximationRank); _host.Check(Utils.Size(_leftFactorMatrix) == _numberOfRows * _approximationRank, "Unexpected matrix size of a factor matrix (matrix P in LIBMF paper)"); _host.Check(Utils.Size(_rightFactorMatrix) == _numberofColumns * _approximationRank, "Unexpected matrix size of a factor matrix (matrix Q in LIBMF paper)"); Utils.WriteSinglesNoCount(ctx.Writer, _leftFactorMatrix.AsSpan(0, _numberOfRows * _approximationRank)); Utils.WriteSinglesNoCount(ctx.Writer, _rightFactorMatrix.AsSpan(0, _numberofColumns * _approximationRank)); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelStream.Name)); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } // Save custom-provided shapes. Those shapes overwrite shapes loaded from the ONNX model file. int customShapeInfosLength = _options.CustomShapeInfos != null ? _options.CustomShapeInfos.Length : 0; ctx.Writer.Write(customShapeInfosLength); for (int i = 0; i < customShapeInfosLength; ++i) { var info = _options.CustomShapeInfos[i]; ctx.SaveNonEmptyString(info.Name); ctx.Writer.WriteIntArray(info.Shape); } ctx.Writer.Write(_options.RecursionLimit); }
private static void Save(IChannel ch, ModelSaveContext ctx, CodecFactory factory, ref VBuffer <ReadOnlyMemory <char> > values) { Contracts.AssertValue(ch); ch.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // Codec parameterization: A codec parameterization that should be a ReadOnlyMemory codec // int: n, the number of bytes used to write the values // byte[n]: As encoded using the codec // Get the codec from the factory IValueCodec codec; var result = factory.TryGetCodec(new VectorType(TextType.Instance), out codec); ch.Assert(result); ch.Assert(codec.Type.IsVector); ch.Assert(codec.Type.VectorSize == 0); ch.Assert(codec.Type.ItemType.RawType == typeof(ReadOnlyMemory <char>)); IValueCodec <VBuffer <ReadOnlyMemory <char> > > textCodec = (IValueCodec <VBuffer <ReadOnlyMemory <char> > >)codec; factory.WriteCodec(ctx.Writer.BaseStream, codec); using (var mem = new MemoryStream()) { using (var writer = textCodec.OpenWriter(mem)) { writer.Write(ref values); writer.Commit(); } ctx.Writer.WriteByteArray(mem.ToArray()); } // Make this resemble, more or less, the auxiliary output from the TermTransform. // It will differ somewhat due to the vector being possibly sparse. To distinguish // between missing and empty, empties are not written at all, while missings are. var v = values; char[] buffer = null; ctx.SaveTextStream("Terms.txt", writer => { writer.WriteLine("# Number of terms = {0} of length {1}", v.Count, v.Length); foreach (var pair in v.Items()) { var text = pair.Value; if (text.IsEmpty) { continue; } writer.Write("{0}\t", pair.Key); // REVIEW: What about escaping this, *especially* for linebreaks? // Do C# and .NET really have no equivalent to Python's "repr"? :( if (text.IsEmpty) { writer.WriteLine(); continue; } Utils.EnsureSize(ref buffer, text.Length); var span = text.Span; for (int i = 0; i < text.Length; i++) { buffer[i] = span[i]; } writer.WriteLine(buffer, 0, text.Length); } }); }
private protected sealed override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); SaveCore(ctx); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // byte: indicator for frozen models // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var isFrozen = string.IsNullOrEmpty(_savedModelPath); ctx.Writer.WriteBoolByte(isFrozen); if (isFrozen) { var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); } else { ctx.SaveBinaryStream("TFSavedModel", w => { string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories); w.Write(modelFilePaths.Length); foreach (var fullPath in modelFilePaths) { var relativePath = fullPath.Substring(_savedModelPath.Length + 1); w.Write(relativePath); using (var fs = new FileStream(fullPath, FileMode.Open)) { long fileLength = fs.Length; w.Write(fileLength); long actualWritten = fs.CopyRange(w.BaseStream, fileLength); _host.Assert(actualWritten == fileLength); } } }); } _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
public void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); SaveCore(ctx); }
void ICanSaveModel.Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); SaveCore(ctx); }
public void Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _transform.Save(ctx); }
void ICanSaveModel.Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ((ICanSaveModel)_transform).Save(ctx); }
private protected virtual void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); SaveCore(ctx); }