private XGBoostRankingPredictor(IHostEnvironment env, ModelLoadContext ctx) : base(env, LoaderSignature, ctx)
{
}
private RegressionGamPredictor(IHostEnvironment env, ModelLoadContext ctx)
: base(env, LoaderSignature, ctx)
{
}
// Factory method for SignatureLoadRowMapper. private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) => new DateTimeTransformer(env, ctx).MakeRowMapper(inputSchema);
private Stacking(IHostEnvironment env, ModelLoadContext ctx)
: base(env, LoaderSignature, ctx)
{
}
internal FieldAwareFactorizationMachinePredictionTransformer(IHostEnvironment host, ModelLoadContext ctx)
: base(Contracts.CheckRef(host, nameof(host)).Register(nameof(FieldAwareFactorizationMachinePredictionTransformer)), ctx)
{
// *** Binary format ***
// <base info>
// ids of strings: feature columns.
// float: scorer threshold
// id of string: scorer threshold column
// count of feature columns. FAFM uses more than one.
int featCount = Model.FieldCount;
var featureColumns = new string[featCount];
var featureColumnTypes = new DataViewType[featCount];
for (int i = 0; i < featCount; i++)
{
featureColumns[i] = ctx.LoadString();
if (!TrainSchema.TryGetColumnIndex(featureColumns[i], out int col))
{
throw Host.ExceptSchemaMismatch(nameof(FeatureColumns), "feature", featureColumns[i]);
}
featureColumnTypes[i] = TrainSchema[col].Type;
}
FeatureColumns = featureColumns;
FeatureColumnTypes = featureColumnTypes;
_threshold = ctx.Reader.ReadSingle();
_thresholdColumn = ctx.LoadString();
BindableMapper = ScoreUtils.GetSchemaBindableMapper(Host, Model);
var schema = GetSchema();
var args = new BinaryClassifierScorer.Arguments {
Threshold = _threshold, ThresholdColumn = _thresholdColumn
};
Scorer = new BinaryClassifierScorer(Host, args, new EmptyDataView(Host, TrainSchema), BindableMapper.Bind(Host, schema), schema);
}
private SchemaBindableBinaryPredictorWrapper(IHostEnvironment env, ModelLoadContext ctx)
: base(env, ctx)
{
CheckValid(out _distMapper);
}
private RankerPerInstanceTransform(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
{
_transform = new Transform(env, ctx, input);
}
public ImplOne(IHostEnvironment env, ModelLoadContext ctx, string scoreColumnKind)
: base(env, ctx, scoreColumnKind)
{
}
public ImplOneWithCalibrator(IHostEnvironment env, ModelLoadContext ctx, string scoreColumnKind)
: base(env, ctx, scoreColumnKind)
{
Host.Assert(_scoreColumnKind == MetadataUtils.Const.ScoreColumnKind.BinaryClassification);
CheckBinaryLabel(false, Host, PredictorModels);
}
private LightGbmRankingModelParameters(IHostEnvironment env, ModelLoadContext ctx)
: base(env, RegistrationName, ctx, GetVersionInfo())
{
}
private static LightGbmRankingModelParameters Create(IHostEnvironment env, ModelLoadContext ctx)
{
return(new LightGbmRankingModelParameters(env, ctx));
}
public NearestNeighborsValueMapper(IHost host, ModelLoadContext ctx)
{
_host = host;
_host.CheckValue(ctx, "ctx");
ReadCore(ctx);
}
// Factory method for SignatureLoadRowMapper. private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, ISchema inputSchema) => Create(env, ctx).MakeRowMapper(inputSchema);
protected BaseMultiAverager(IHostEnvironment env, string name, ModelLoadContext ctx)
: base(env, name, ctx)
{
}
internal RankingPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(RankingPredictionTransformer <TModel>)), ctx)
{
Scorer = GetGenericScorer();
}
public static SchemaBindablePipelineEnsembleBase Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
var scoreColumnKind = ctx.LoadNonEmptyString();
switch (scoreColumnKind)
{
case MetadataUtils.Const.ScoreColumnKind.BinaryClassification:
return(new ImplOneWithCalibrator(env, ctx, scoreColumnKind));
case MetadataUtils.Const.ScoreColumnKind.Regression:
case MetadataUtils.Const.ScoreColumnKind.AnomalyDetection:
return(new ImplOne(env, ctx, scoreColumnKind));
case MetadataUtils.Const.ScoreColumnKind.MultiClassClassification:
return(new ImplVec(env, ctx, scoreColumnKind));
default:
throw env.Except("Unknown score kind");
}
}
private SchemaBindablePredictorWrapper(IHostEnvironment env, ModelLoadContext ctx)
: base(env, ctx)
{
_scoreColumnKind = GetScoreColumnKind(Predictor);
}
private static FastForestRegressionModelParameters Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
return(new FastForestRegressionModelParameters(env, ctx));
}
public static SchemaBindableQuantileRegressionPredictor Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
return(new SchemaBindableQuantileRegressionPredictor(env, ctx));
}
private FastTreeTweedieModelParameters(IHostEnvironment env, ModelLoadContext ctx)
: base(env, RegistrationName, ctx, GetVersionInfo())
{
}
private static MultiClassNaiveBayesModelParameters Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
return(new MultiClassNaiveBayesModelParameters(env, ctx));
}
private LightGbmBinaryPredictor(IHostEnvironment env, ModelLoadContext ctx)
: base(env, RegistrationName, ctx, GetVersionInfo())
{
}
private static FieldAwareFactorizationMachineModelParameters Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
return(new FieldAwareFactorizationMachineModelParameters(env, ctx));
}
public LdaState(IExceptionContext ectx, ModelLoadContext ctx)
: this()
{
ectx.AssertValue(ctx);
// *** Binary format ***
// <ColInfoEx>
// int: vocabnum
// long: memblocksize
// long: aliasMemBlockSize
// (serializing term by term, for one term)
// int: term_id, int: topic_num, KeyValuePair<int, int>[]: termTopicVector
InfoEx = new ColInfoEx(ectx, ctx);
_numVocab = ctx.Reader.ReadInt32();
ectx.CheckDecode(_numVocab > 0);
long memBlockSize = ctx.Reader.ReadInt64();
ectx.CheckDecode(memBlockSize > 0);
long aliasMemBlockSize = ctx.Reader.ReadInt64();
ectx.CheckDecode(aliasMemBlockSize > 0);
_ldaTrainer = new LdaSingleBox(
InfoEx.NumTopic,
_numVocab, /* Need to set number of vocabulary here */
InfoEx.AlphaSum,
InfoEx.Beta,
InfoEx.NumIter,
InfoEx.LikelihoodInterval,
InfoEx.NumThread,
InfoEx.MHStep,
InfoEx.NumSummaryTermPerTopic,
false,
InfoEx.NumMaxDocToken);
_ldaTrainer.AllocateModelMemory(_numVocab, InfoEx.NumTopic, memBlockSize, aliasMemBlockSize);
for (int i = 0; i < _numVocab; i++)
{
int termID = ctx.Reader.ReadInt32();
ectx.CheckDecode(termID >= 0);
int termTopicNum = ctx.Reader.ReadInt32();
ectx.CheckDecode(termTopicNum >= 0);
int[] topicId = new int[termTopicNum];
int[] topicProb = new int[termTopicNum];
for (int j = 0; j < termTopicNum; j++)
{
topicId[j] = ctx.Reader.ReadInt32();
topicProb[j] = ctx.Reader.ReadInt32();
}
//set the topic into _ldaTrainer inner topic table
_ldaTrainer.SetModel(termID, topicId, topicProb, termTopicNum);
}
//do the preparation
if (!_predictionPreparationDone)
{
_ldaTrainer.InitializeBeforeTest();
_predictionPreparationDone = true;
}
}
private static FieldAwareFactorizationMachinePredictionTransformer Create(IHostEnvironment env, ModelLoadContext ctx) => new FieldAwareFactorizationMachinePredictionTransformer(env, ctx);
private static void LoadPredictors <TPredictor>(IHostEnvironment env, TPredictor[] predictors, ModelLoadContext ctx)
where TPredictor : class
{
for (int i = 0; i < predictors.Length; i++)
{
ctx.LoadModel <TPredictor, SignatureLoadModel>(env, out predictors[i], string.Format(SubPredictorFmt, i));
}
}
private FastForestClassificationModelParameters(IHostEnvironment env, ModelLoadContext ctx)
: base(env, RegistrationName, ctx, GetVersionInfo())
{
}
private FastTreeRegressionPredictor(IHostEnvironment env, ModelLoadContext ctx)
: base(env, RegistrationName, ctx, GetVersionInfo())
{
}
private MultiClassClassifierScorer(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, ctx, input, OutputTypeMatches, GetPredColType)
{
// *** Binary format ***
// <base info>
}
// Factory method for SignatureLoadModel.
private static TensorFlowTransform Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
// *** Binary format ***
// byte: indicator for frozen models
// stream: tensorFlow model.
// int: number of input columns
// for each input column
// int: id of int column name
// int: number of output columns
// for each output column
// int: id of output column name
GetModelInfo(env, ctx, out string[] inputs, out string[] outputs, out bool isFrozen);
if (isFrozen)
{
byte[] modelBytes = null;
if (!ctx.TryLoadBinaryStream("TFModel", r => modelBytes = r.ReadByteArray()))
{
throw env.ExceptDecode();
}
return(new TensorFlowTransform(env, TensorFlowUtils.LoadTFSession(env, modelBytes), inputs, outputs, null, false));
}
var tempDirPath = Path.GetFullPath(Path.Combine(Path.GetTempPath(), RegistrationName + "_" + Guid.NewGuid()));
TensorFlowUtils.CreateFolderWithAclIfNotExists(env, tempDirPath);
try
{
var load = ctx.TryLoadBinaryStream("TFSavedModel", br =>
{
int count = br.ReadInt32();
for (int n = 0; n < count; n++)
{
string relativeFile = br.ReadString();
long fileLength = br.ReadInt64();
string fullFilePath = Path.Combine(tempDirPath, relativeFile);
string fullFileDir = Path.GetDirectoryName(fullFilePath);
if (fullFileDir != tempDirPath)
{
TensorFlowUtils.CreateFolderWithAclIfNotExists(env, fullFileDir);
}
using (var fs = new FileStream(fullFilePath, FileMode.Create, FileAccess.Write))
{
long actualRead = br.BaseStream.CopyRange(fs, fileLength);
env.Assert(actualRead == fileLength);
}
}
});
return(new TensorFlowTransform(env, TensorFlowUtils.GetSession(env, tempDirPath), inputs, outputs, tempDirPath, true));
}
catch (Exception)
{
TensorFlowUtils.DeleteFolderWithRetries(env, tempDirPath);
throw;
}
}
