public override void Save(ModelSaveContext ctx) { // *** Binary format *** // (base fields) // if version >= VersionAddedAliases // foreach column: // foreach non-null alias // int: index of the alias // int: string id of the alias // int: -1, marks the end of the list base.Save(ctx); Contracts.Assert(_aliases.Length == Infos.Length); for (int i = 0; i < Infos.Length; i++) { Contracts.Assert(_aliases[i].Length == Infos[i].SrcIndices.Length); for (int j = 0; j < _aliases[i].Length; j++) { if (!string.IsNullOrEmpty(_aliases[i][j])) { ctx.Writer.Write(j); ctx.SaveNonEmptyString(_aliases[i][j]); } } ctx.Writer.Write(-1); } }
private static void SaveCore(ModelSaveContext ctx, Action <ModelSaveContext> loaderSaveAction, TransformEx[] transforms) { Contracts.AssertValue(ctx); Contracts.AssertValue(loaderSaveAction); Contracts.AssertValueOrNull(transforms); // *** Binary format *** // int: sizeof(Float) // int: number of transforms // foreach transform: (starting from version VersionAddedTags) // string: tag // string: args string ctx.Writer.Write(sizeof(Float)); ctx.Writer.Write(transforms.Length); using (var loaderCtx = new ModelSaveContext(ctx.Repository, Path.Combine(ctx.Directory ?? "", "Loader"), ModelLoadContext.ModelStreamName)) { loaderSaveAction(loaderCtx); loaderCtx.Done(); } for (int i = 0; i < transforms.Length; i++) { var dirName = string.Format(TransformDirTemplate, i); ctx.SaveModel(transforms[i].Transform, dirName); Contracts.AssertNonEmpty(transforms[i].Tag); ctx.SaveNonEmptyString(transforms[i].Tag); ctx.SaveStringOrNull(transforms[i].ArgsString); } }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: number of added columns // for each added column // int: id of output column name // byte: useCounter // if !useCounter // uint: seed0 // uint: seed1 // uint: seed2 // uint: seed3 int size = InfoCount; ctx.Writer.Write(size); for (int i = 0; i < size; i++) { ctx.SaveNonEmptyString(GetColumnNameCore(i)); ctx.Writer.WriteBoolByte(UseCounter[i]); if (!UseCounter[i]) { States[i].Save(ctx.Writer); } } }
internal void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: number of added columns // for each added column // int: id of output column name // int: id of input column name ctx.Writer.Write(Infos.Length); foreach (var info in Infos) { ctx.SaveNonEmptyString(info.Name); ctx.SaveNonEmptyString(Input[info.Source].Name); } }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: id of output column name ctx.SaveNonEmptyString(GetColumnNameCore(0)); }
public virtual void Save(ModelSaveContext ctx) { // *** Binary format ** // int: Id of the score column name // int: Id of the label column name ctx.SaveNonEmptyString(ScoreCol); ctx.SaveStringOrNull(LabelCol); }
public override void Save(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(Model.ToByteArray()); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) ctx.SaveNonEmptyString(colName); Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) ctx.SaveNonEmptyString(colName); }
protected void SaveBase(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: id of the suffix // int: the number of input column roles // for each input column: // int: id of the column role // int: id of the column name ctx.SaveString(Suffix); var cols = Mapper.GetInputColumnRoles().ToArray(); ctx.Writer.Write(cols.Length); foreach (var kvp in cols) { ctx.SaveNonEmptyString(kvp.Key.Value); ctx.SaveNonEmptyString(kvp.Value); } }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: count of group column infos (ie, count of source columns) // For each group column info // int: the tokenizer language // int: the id of source column name // int: the id of languages column name // bool: whether the types output is required // For each column info that belongs to this group column info // (either one column info for tokens or two for tokens and types) // int: the id of the column name Contracts.Assert(Utils.Size(GroupInfos) > 0); ctx.Writer.Write(GroupInfos.Length); int iinfo = 0; for (int groupId = 0; groupId < GroupInfos.Length; groupId++) { var groupInfo = GroupInfos[groupId]; Contracts.Assert(Enum.IsDefined(typeof(Language), groupInfo.Lang)); ctx.Writer.Write((int)groupInfo.Lang); ctx.SaveNonEmptyString(groupInfo.SrcColName); ctx.SaveStringOrNull(groupInfo.LangsColName); ctx.Writer.WriteBoolByte(groupInfo.RequireTypes); int count = groupInfo.RequireTypes ? 2 : 1; int lim = iinfo + count; for (; iinfo < lim; iinfo++) { Contracts.Assert(Infos[iinfo].GroupInfoId == groupId); ctx.SaveNonEmptyString(GetColumnNameCore(iinfo)); } } Contracts.Assert(iinfo == Infos.Length); }
public virtual void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); Host.Assert(InitialWindowSize >= 0); Host.Assert(WindowSize >= 0); // *** Binary format *** // int: _windowSize // int: _initialWindowSize // int (string ID): _inputColumnName // int (string ID): _outputColumnName // ColumnType: _transform.Schema.GetColumnType(0) ctx.Writer.Write(WindowSize); ctx.Writer.Write(InitialWindowSize); ctx.SaveNonEmptyString(InputColumnName); ctx.SaveNonEmptyString(OutputColumnName); var bs = new BinarySaver(Host, new BinarySaver.Arguments()); bs.TryWriteTypeDescription(ctx.Writer.BaseStream, OutputColumnType, out int byteWritten); }
public virtual void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: number of added columns // for each added column // int: id of output column name // int: number of input column names // int[]: ids of input column names ctx.Writer.Write(Infos.Length); for (int i = 0; i < Infos.Length; i++) { var info = Infos[i]; ctx.SaveNonEmptyString(GetColumnNameCore(i)); ctx.Writer.Write(info.SrcIndices.Length); foreach (int src in info.SrcIndices) { ctx.SaveNonEmptyString(Input.GetColumnName(src)); } } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: number of output columns // for each output column: // int: number of inputs // foreach input // int: Id of the input column name // int: Id of the expression // int: Id of the output column name // int: the index of the vector input (or -1) // int[]: The data kinds of the input columns ctx.Writer.Write(_columns.Length); for (int i = 0; i < _columns.Length; i++) { var inputColumnNames = _columns[i].InputColumnNames; ctx.Writer.Write(inputColumnNames.Length); for (int j = 0; j < inputColumnNames.Length; j++) { ctx.SaveNonEmptyString(inputColumnNames[j]); } ctx.SaveNonEmptyString(_columns[i].Expression); ctx.SaveNonEmptyString(_columns[i].OutputColumnName); Host.Assert(_columns[i].VectorInputColumn >= -1); ctx.Writer.Write(_columns[i].VectorInputColumn); for (int j = 0; j < _columns[i].InputKinds.Length; j++) { ctx.Writer.Write(_columns[i].InputKinds[j].ToIndex()); } } }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); var buffer = new TFBuffer(); _session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); Contracts.AssertNonEmpty(_inputColNames); ctx.Writer.Write(_inputColNames.Length); foreach (var colName in _inputColNames) { ctx.SaveNonEmptyString(colName); } ctx.SaveNonEmptyString(_outputColName); }
public void Save(ModelSaveContext ctx) { _ectx.AssertValue(ctx); // *** Binary format *** // int: ungroup mode // int: K - number of pivot columns // int[K]: ids of pivot column names ctx.Writer.Write((int)Mode); ctx.Writer.Write(_infos.Length); foreach (var ex in _infos) ctx.SaveNonEmptyString(ex.Name); }
internal void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: number of columns // foreach column: // int: id of column name // byte: DataKind // byte: bool of whether this is a key type // for a key type: // ulong: count for key range // int: number of segments // foreach segment: // string id: name // int: min // int: lim // byte: force vector (verWrittenCur: verIsVectorSupported) ctx.Writer.Write(Infos.Length); for (int iinfo = 0; iinfo < Infos.Length; iinfo++) { var info = Infos[iinfo]; ctx.SaveNonEmptyString(info.Name); var type = info.ColType.GetItemType(); InternalDataKind rawKind = type.GetRawKind(); Contracts.Assert((InternalDataKind)(byte)rawKind == rawKind); ctx.Writer.Write((byte)rawKind); ctx.Writer.WriteBoolByte(type is KeyDataViewType); if (type is KeyDataViewType key) { ctx.Writer.Write(key.Count); } if (info.Segments is null) { ctx.Writer.Write(0); } else { ctx.Writer.Write(info.Segments.Length); foreach (var seg in info.Segments) { ctx.SaveStringOrNull(seg.Name); ctx.Writer.Write(seg.Min); ctx.Writer.Write(seg.Lim); ctx.Writer.WriteBoolByte(seg.ForceVector); } } } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelFile)); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // byte: default HiddenColumnOption value // int: number of raw column infos // for each raw column info // int: id of output column name // int: id of input column name // byte: HiddenColumnOption Contracts.Assert((HiddenColumnOption)(byte)HidDefault == HidDefault); ctx.Writer.Write((byte)HidDefault); ctx.Writer.Write(RawInfos.Length); for (int i = 0; i < RawInfos.Length; i++) { var raw = RawInfos[i]; ctx.SaveNonEmptyString(raw.Name); ctx.SaveNonEmptyString(raw.Source); Contracts.Assert((HiddenColumnOption)(byte)raw.Hid == raw.Hid); ctx.Writer.Write((byte)raw.Hid); } }
public void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // byte[]: A chunk of data saving both the type and value of the label names, as saved by the BinarySaver. // int: string id of the metadata kind ctx.SaveModel(_bindable, _innerDir); Utils.MarshalActionInvoke(SaveCore <int>, _type.ItemType.RawType, ctx); ctx.SaveNonEmptyString(_metadataKind); }
private protected virtual void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); Host.Assert(InitialWindowSize >= 0); Host.Assert(WindowSize >= 0); // *** Binary format *** // int: _windowSize // int: _initialWindowSize // int (string ID): _sourceColumnName // int (string ID): _outputColumnName // ColumnType: _transform.Schema.GetColumnType(0) ctx.Writer.Write(WindowSize); ctx.Writer.Write(InitialWindowSize); ctx.SaveNonEmptyString(InputColumnName); ctx.SaveNonEmptyString(OutputColumnName); ctx.Writer.Write(ForecastingConfidenceIntervalMinOutputColumnName ?? string.Empty); ctx.Writer.Write(ForecastingConfidenceIntervalMaxOutputColumnName ?? string.Empty); var bs = new BinarySaver(Host, new BinarySaver.Arguments()); bs.TryWriteTypeDescription(ctx.Writer.BaseStream, OutputColumnType, out int byteWritten); }
public void Save(ModelSaveContext ctx) { ctx.SetVersionInfo(GetVersionInfo()); ctx.Writer.WriteBoolByte(KeepColumns); ctx.Writer.WriteBoolByte(KeepHidden); ctx.Writer.WriteBoolByte(IgnoreMissing); var length = _selectedColumns.Length; ctx.Writer.Write(length); for (int i = 0; i < length; i++) { ctx.SaveNonEmptyString(_selectedColumns[i]); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); ctx.SaveBinaryStream("OnnxModel", w => { w.WriteByteArray(File.ReadAllBytes(Model.ModelStream.Name)); }); Host.CheckNonEmpty(Inputs, nameof(Inputs)); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } Host.CheckNonEmpty(Outputs, nameof(Outputs)); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } // Save custom-provided shapes. Those shapes overwrite shapes loaded from the ONNX model file. int customShapeInfosLength = _options.CustomShapeInfos != null ? _options.CustomShapeInfos.Length : 0; ctx.Writer.Write(customShapeInfosLength); for (int i = 0; i < customShapeInfosLength; ++i) { var info = _options.CustomShapeInfos[i]; ctx.SaveNonEmptyString(info.Name); ctx.Writer.WriteIntArray(info.Shape); } ctx.Writer.Write(_options.RecursionLimit); }
public void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // bool: whether to keep (vs drop) the named columns // int: number of names // int[]: the ids of the names ctx.Writer.WriteBoolByte(Keep); ctx.Writer.Write(Names.Count); foreach (var name in Names) { ctx.SaveNonEmptyString(name); } }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }
public override void Save(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // int: number of columns // int[]: ids of column names ctx.Writer.Write(sizeof(Float)); Host.Assert(_infos.Length > 0); ctx.Writer.Write(_infos.Length); foreach (var info in _infos) { ctx.SaveNonEmptyString(Source.Schema.GetColumnName(info.Index)); } }
public override void Save(ModelSaveContext ctx) { Contracts.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format ** // base // int: _scoreSize // int[]: Ids of the quantile names base.Save(ctx); Host.Assert(_scoreSize > 0); ctx.Writer.Write(_scoreSize); for (int i = 0; i < _scoreSize; i++) { ctx.SaveNonEmptyString(_quantiles[i].ToString()); } }
private protected override void SaveModel(ModelSaveContext ctx) { Host.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // int: sizeof(Float) // int: id of column name // double: min // double: max // byte: complement // byte: includeMin // byte: includeMax ctx.Writer.Write(sizeof(float)); ctx.SaveNonEmptyString(Source.Schema[_index].Name); Host.Assert(_min < _max); ctx.Writer.Write(_min); ctx.Writer.Write(_max); ctx.Writer.WriteBoolByte(_complement); ctx.Writer.WriteBoolByte(_includeMin); ctx.Writer.WriteBoolByte(_includeMax); }
internal void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // int: number of columns // foreach column: // int: id of column name // byte: DataKind // byte: bool of whether this is a key type // for a key type: // ulong: count for key range // byte: bool of whether the source index is valid // for a valid source index: // int: source index ctx.Writer.Write(Infos.Length); for (int iinfo = 0; iinfo < Infos.Length; iinfo++) { var info = Infos[iinfo]; ctx.SaveNonEmptyString(info.Name); var type = info.ColType.GetItemType(); InternalDataKind rawKind = type.GetRawKind(); Contracts.Assert((InternalDataKind)(byte)rawKind == rawKind); ctx.Writer.Write((byte)rawKind); ctx.Writer.WriteBoolByte(type is KeyDataViewType); if (type is KeyDataViewType key) { ctx.Writer.Write(key.Count); } ctx.Writer.WriteBoolByte(info.SourceIndex.HasValue); if (info.SourceIndex.HasValue) { ctx.Writer.Write(info.SourceIndex.GetValueOrDefault()); } } }
public void Save(IHostEnvironment env, ModelSaveContext ctx) { Contracts.AssertValue(ctx); // *** Binary format *** // Schema of the data view containing the optional columns // int: number of added columns // for each added column // int: id of output column name // ColumnType: the type of the column var noRows = new EmptyDataView(env, _inputWithOptionalColumn); var saverArgs = new BinarySaver.Arguments(); saverArgs.Silent = true; var saver = new BinarySaver(env, saverArgs); using (var strm = new MemoryStream()) { saver.SaveData(strm, noRows, _srcColsWithOptionalColumn); ctx.SaveBinaryStream("Schema.idv", w => w.WriteByteArray(strm.ToArray())); } int size = InfoCount; ctx.Writer.Write(size); saver = new BinarySaver(env, new BinarySaver.Arguments()); for (int i = 0; i < size; i++) { ctx.SaveNonEmptyString(GetColumnNameCore(i)); var columnType = ColumnTypes[i]; int written; saver.TryWriteTypeDescription(ctx.Writer.BaseStream, columnType, out written); } }
public void Save(ModelSaveContext ctx) { _host.AssertValue(ctx); ctx.CheckAtModel(); ctx.SetVersionInfo(GetVersionInfo()); // *** Binary format *** // byte: indicator for frozen models // stream: tensorFlow model. // int: number of input columns // for each input column // int: id of int column name // int: number of output columns // for each output column // int: id of output column name var isFrozen = string.IsNullOrEmpty(_savedModelPath); ctx.Writer.WriteBoolByte(isFrozen); if (isFrozen) { var buffer = new TFBuffer(); Session.Graph.ToGraphDef(buffer); ctx.SaveBinaryStream("TFModel", w => { w.WriteByteArray(buffer.ToArray()); }); } else { ctx.SaveBinaryStream("TFSavedModel", w => { string[] modelFilePaths = Directory.GetFiles(_savedModelPath, "*", SearchOption.AllDirectories); w.Write(modelFilePaths.Length); foreach (var fullPath in modelFilePaths) { var relativePath = fullPath.Substring(_savedModelPath.Length + 1); w.Write(relativePath); using (var fs = new FileStream(fullPath, FileMode.Open)) { long fileLength = fs.Length; w.Write(fileLength); long actualWritten = fs.CopyRange(w.BaseStream, fileLength); _host.Assert(actualWritten == fileLength); } } }); } _host.AssertNonEmpty(Inputs); ctx.Writer.Write(Inputs.Length); foreach (var colName in Inputs) { ctx.SaveNonEmptyString(colName); } _host.AssertNonEmpty(Outputs); ctx.Writer.Write(Outputs.Length); foreach (var colName in Outputs) { ctx.SaveNonEmptyString(colName); } }