private protected override void SaveCore(ModelSaveContext ctx) { base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: _labelCount // int[_labelCount]: _labelHistogram // int: _featureCount // int[_labelCount][_featureCount]: _featureHistogram // int[_labelCount]: _absentFeaturesLogProb ctx.Writer.WriteIntArray(_labelHistogram.AsSpan(0, _labelCount)); ctx.Writer.Write(_featureCount); for (int i = 0; i < _labelCount; i += 1) { if (_labelHistogram[i] > 0) { ctx.Writer.WriteIntsNoCount(_featureHistogram[i].AsSpan(0, _featureCount)); } } ctx.Writer.WriteDoublesNoCount(_absentFeaturesLogProb.AsSpan(0, _labelCount)); }
public override void Save(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // int: id of column name // double: min // double: max // byte: complement // byte: includeMin // byte: includeMax ctx.Writer.Write(sizeof(float)); ctx.SaveNonEmptyString(Source.Schema[_index].Name); Host.Assert(_min < _max); ctx.Writer.Write(_min); ctx.Writer.Write(_max); ctx.Writer.WriteBoolByte(_complement); ctx.Writer.WriteBoolByte(_includeMin); ctx.Writer.WriteBoolByte(_includeMax); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var buffer = new TFBuffer(); _session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); Contracts.AssertNonEmpty(_inputColNames); ctx.Writer.Write(_inputColNames.Length); foreach (var colName in _inputColNames) { ctx.SaveNonEmptyString(colName); } ctx.SaveNonEmptyString(_outputColName); }
/// <summary> /// Save model to the given context /// </summary> void ICanSaveModel.Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of rows (m), the limit on row // int: number of columns (n), the limit on column // int: rank of factor matrices (k) // float[m * k]: the left factor matrix // float[k * n]: the right factor matrix _host.Check(NumberOfRows > 0, "Number of rows must be positive"); _host.Check(NumberOfColumns > 0, "Number of columns must be positive"); _host.Check(ApproximationRank > 0, "Number of latent factors must be positive"); ctx.Writer.Write(NumberOfRows); ctx.Writer.Write(NumberOfColumns); ctx.Writer.Write(ApproximationRank); _host.Check(Utils.Size(_leftFactorMatrix) == NumberOfRows * ApproximationRank, "Unexpected matrix size of a factor matrix (matrix P in LIBMF paper)"); _host.Check(Utils.Size(_rightFactorMatrix) == NumberOfColumns * ApproximationRank, "Unexpected matrix size of a factor matrix (matrix Q in LIBMF paper)"); Utils.WriteSinglesNoCount(ctx.Writer, _leftFactorMatrix); Utils.WriteSinglesNoCount(ctx.Writer, _rightFactorMatrix); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelStream.Name)); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } // Save custom-provided shapes. Those shapes overwrite shapes loaded from the ONNX model file. int customShapeInfosLength = _options.CustomShapeInfos != null ? _options.CustomShapeInfos.Length : 0; ctx.Writer.Write(customShapeInfosLength); for (int i = 0; i < customShapeInfosLength; ++i) { var info = _options.CustomShapeInfos[i]; ctx.SaveNonEmptyString(info.Name); ctx.Writer.WriteIntArray(info.Shape); } ctx.Writer.Write(_options.RecursionLimit); }
public override void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.SetVersionInfo(GetVersionInfo()); base.Save(ctx); // *** Binary format *** // foreach of the _labelCardinality dictionaries // int: number N of elements in the dictionary. // for each of the N elements: // long: key // Single: value foreach (var table in Tables) { ctx.Writer.Write(table.Count); foreach (var pair in table) { ctx.Writer.Write(pair.Key); Contracts.Assert(pair.Value >= 0); ctx.Writer.Write(pair.Value); } } }
public void Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); _transform.Save(ctx); }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); ctx.SetVersionInfo(GetVersionInfo()); SaveCore(ctx); }
protected override void SaveCore(ModelSaveContext ctx) { Contracts.Assert(!Normalize); base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); }
void ICanSaveModel.Save(ModelSaveContext ctx) { ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ((ICanSaveModel)_transform).Save(ctx); }
private protected override void SaveCore(ModelSaveContext ctx) { base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // byte: indicator for frozen models // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var isFrozen = string.IsNullOrEmpty(_savedModelPath); ctx.Writer.WriteBoolByte(isFrozen); if (isFrozen) { var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); } else { ctx.SaveBinaryStream("TFSavedModel", w => { string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories); w.Write(modelFilePaths.Length); foreach (var fullPath in modelFilePaths) { var relativePath = fullPath.Substring(_savedModelPath.Length + 1); w.Write(relativePath); using (var fs = new FileStream(fullPath, FileMode.Open)) { long fileLength = fs.Length; w.Write(fileLength); long actualWritten = fs.CopyRange(w.BaseStream, fileLength); _host.Assert(actualWritten == fileLength); } } }); } _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
private protected override void SaveCore(ModelSaveContext ctx) { Contracts.AssertValue(ctx); ctx.SetVersionInfo(GetVersionInfo()); _bindings.SaveModel(ctx); }
protected override void SaveCore(ModelSaveContext ctx) { base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); Host.Assert(_biases.Length == _numClasses); Host.Assert(_biases.Length == _weights.Length); #if DEBUG foreach (var fw in _weights) { Host.Assert(fw.Length == _numFeatures); } #endif // *** Binary format *** // int: number of features // int: number of classes = number of biases // float[]: biases // (weight matrix, in CSR if sparse) // (see https://netlib.org/linalg/html_templates/node91.html#SECTION00931100000000000000) // int: number of row start indices (_numClasses + 1 if sparse, 0 if dense) // int[]: row start indices // int: total number of column indices (0 if dense) // int[]: column index of each non-zero weight // int: total number of non-zero weights (same as number of column indices if sparse, num of classes * num of features if dense) // float[]: non-zero weights // bool: whether label names are present // int[]: Id of label names (optional, in a separate stream) // LinearModelStatistics: model statistics (optional, in a separate stream) ctx.Writer.Write(_numFeatures); ctx.Writer.Write(_numClasses); ctx.Writer.WriteFloatsNoCount(_biases, _numClasses); // _weights == _weighsDense means we checked that all vectors in _weights // are actually dense, and so we assigned the same object, or it came dense // from deserialization. if (_weights == _weightsDense) { ctx.Writer.Write(0); // Number of starts. ctx.Writer.Write(0); // Number of indices. ctx.Writer.Write(_numFeatures * _weights.Length); foreach (var fv in _weights) { Host.Assert(fv.Length == _numFeatures); ctx.Writer.WriteFloatsNoCount(fv.Values, _numFeatures); } } else { // Number of starts. ctx.Writer.Write(_numClasses + 1); // Starts always starts with 0. int numIndices = 0; ctx.Writer.Write(numIndices); for (int i = 0; i < _weights.Length; i++) { // REVIEW: Assuming the presence of *any* zero justifies // writing in sparse format seems stupid, but might be difficult // to change without changing the format since the presence of // any sparse vector means we're writing indices anyway. Revisit. // This is actually a bug waiting to happen: sparse/dense vectors // can have different dot products even if they are logically the // same vector. numIndices += NonZeroCount(ref _weights[i]); ctx.Writer.Write(numIndices); } ctx.Writer.Write(numIndices); { // just scoping the count so we can use another further down int count = 0; foreach (var fw in _weights) { if (fw.IsDense) { for (int i = 0; i < fw.Length; i++) { if (fw.Values[i] != 0) { ctx.Writer.Write(i); count++; } } } else { ctx.Writer.WriteIntsNoCount(fw.Indices, fw.Count); count += fw.Count; } } Host.Assert(count == numIndices); } ctx.Writer.Write(numIndices); { int count = 0; foreach (var fw in _weights) { if (fw.IsDense) { for (int i = 0; i < fw.Length; i++) { if (fw.Values[i] != 0) { ctx.Writer.Write(fw.Values[i]); count++; } } } else { ctx.Writer.WriteFloatsNoCount(fw.Values, fw.Count); count += fw.Count; } } Host.Assert(count == numIndices); } } Contracts.AssertValueOrNull(_labelNames); if (_labelNames != null) { ctx.SaveBinaryStream(LabelNamesSubModelFilename, w => SaveLabelNames(ctx, w)); } Contracts.AssertValueOrNull(_stats); if (_stats != null) { using (var statsCtx = new ModelSaveContext(ctx.Repository, Path.Combine(ctx.Directory ?? "", ModelStatsSubModelFilename), ModelLoadContext.ModelStreamName)) { _stats.Save(statsCtx); statsCtx.Done(); } } }
private protected override void SaveModel(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.SetVersionInfo(GetVersionInfo()); base.SaveModel(ctx); }
public override void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.SetVersionInfo(GetVersionInfo()); base.Save(ctx); }
private static void Save(IChannel ch, ModelSaveContext ctx, CodecFactory factory, ref VBuffer <ReadOnlyMemory <char> > values) { Contracts.AssertValue(ch); ch.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // Codec parameterization: A codec parameterization that should be a ReadOnlyMemory codec // int: n, the number of bytes used to write the values // byte[n]: As encoded using the codec // Get the codec from the factory IValueCodec codec; var result = factory.TryGetCodec(new VectorType(TextType.Instance), out codec); ch.Assert(result); ch.Assert(codec.Type.IsVector); ch.Assert(codec.Type.VectorSize == 0); ch.Assert(codec.Type.ItemType.RawType == typeof(ReadOnlyMemory <char>)); IValueCodec <VBuffer <ReadOnlyMemory <char> > > textCodec = (IValueCodec <VBuffer <ReadOnlyMemory <char> > >)codec; factory.WriteCodec(ctx.Writer.BaseStream, codec); using (var mem = new MemoryStream()) { using (var writer = textCodec.OpenWriter(mem)) { writer.Write(ref values); writer.Commit(); } ctx.Writer.WriteByteArray(mem.ToArray()); } // Make this resemble, more or less, the auxiliary output from the TermTransform. // It will differ somewhat due to the vector being possibly sparse. To distinguish // between missing and empty, empties are not written at all, while missings are. var v = values; char[] buffer = null; ctx.SaveTextStream("Terms.txt", writer => { writer.WriteLine("# Number of terms = {0} of length {1}", v.Count, v.Length); foreach (var pair in v.Items()) { var text = pair.Value; if (text.IsEmpty) { continue; } writer.Write("{0}\t", pair.Key); // REVIEW: What about escaping this, *especially* for linebreaks? // Do C# and .NET really have no equivalent to Python's "repr"? :( if (text.IsEmpty) { writer.WriteLine(); continue; } Utils.EnsureSize(ref buffer, text.Length); var span = text.Span; for (int i = 0; i < text.Length; i++) { buffer[i] = span[i]; } writer.WriteLine(buffer, 0, text.Length); } }); }