private TDistPredictor TrainOne(IChannel ch, TScalarTrainer trainer, RoleMappedData data, int cls1, int cls2)
{
string dstName;
var view = MapLabels(data, cls1, cls2, out dstName);
var roles = data.Schema.GetColumnRoleNames()
.Where(kvp => kvp.Key.Value != CR.Label.Value)
.Prepend(CR.Label.Bind(dstName));
var td = RoleMappedData.Create(view, roles);
trainer.Train(td);
ICalibratorTrainer calibrator;
if (!Args.Calibrator.IsGood())
{
calibrator = null;
}
else
{
calibrator = Args.Calibrator.CreateInstance(Host);
}
TScalarPredictor predictor = trainer.CreatePredictor();
var res = CalibratorUtils.TrainCalibratorIfNeeded(Host, ch, calibrator, Args.MaxCalibrationExamples,
trainer, predictor, td);
var dist = res as TDistPredictor;
Host.Check(dist != null, "Calibrated predictor does not implement the expected interface");
Host.Check(dist is IValueMapperDist, "Calibrated predictor does not implement the IValueMapperDist interface");
return(dist);
}
public virtual void CalculateMetrics(FeatureSubsetModel <IPredictorProducing <TOutput> > model,
ISubsetSelector subsetSelector, Subset subset, Batch batch, bool needMetrics)
{
if (!needMetrics || model == null || model.Metrics != null)
{
return;
}
using (var ch = Host.Start("Calculate metrics"))
{
RoleMappedData testData = subsetSelector.GetTestData(subset, batch);
// Because the training and test datasets are drawn from the same base dataset, the test data role mappings
// are the same as for the train data.
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(model.Predictor, testData, Host, testData.Schema);
// REVIEW: Should we somehow allow the user to customize the evaluator?
// By what mechanism should we allow that?
var evalComp = GetEvaluatorSubComponent();
RoleMappedData scoredTestData = RoleMappedData.Create(scorePipe,
GetColumnRoles(testData.Schema, scorePipe.Schema));
IEvaluator evaluator = evalComp.CreateInstance(Host);
// REVIEW: with the new evaluators, metrics of individual models are no longer
// printed to the Console. Consider adding an option on the combiner to print them.
// REVIEW: Consider adding an option to the combiner to save a data view
// containing all the results of the individual models.
var metricsDict = evaluator.Evaluate(scoredTestData);
if (!metricsDict.TryGetValue(MetricKinds.OverallMetrics, out IDataView metricsView))
{
throw Host.Except("Evaluator did not produce any overall metrics");
}
// REVIEW: We're assuming that the metrics of interest are always doubles here.
var metrics = EvaluateUtils.GetMetrics(metricsView, getVectorMetrics: false);
model.Metrics = metrics.ToArray();
ch.Done();
}
}
/// <summary>
/// Return a dataset with non-selected features zeroed out.
/// </summary>
public static RoleMappedData SelectFeatures(IHost host, RoleMappedData data, BitArray features)
{
Contracts.AssertValue(host);
Contracts.AssertValue(data);
Contracts.Assert(data.Schema.Feature != null);
Contracts.AssertValue(features);
var type = data.Schema.Feature.Type;
Contracts.Assert(features.Length == type.VectorSize);
int card = Utils.GetCardinality(features);
if (card == type.VectorSize)
{
return(data);
}
// REVIEW: This doesn't preserve metadata on the features column. Should it?
var name = data.Schema.Feature.Name;
var view = LambdaColumnMapper.Create(
host, "FeatureSelector", data.Data, name, name, type, type,
(ref VBuffer <Single> src, ref VBuffer <Single> dst) => SelectFeatures(ref src, features, card, ref dst));
var res = RoleMappedData.Create(view, data.Schema.GetColumnRoleNames());
return(res);
}
public override IEnumerable <Subset> GetSubsets(Batch batch, IRandom rand)
{
for (int i = 0; i < Size; i++)
{
// REVIEW: Consider ways to reintroduce "balanced" samples.
var viewTrain = new BootstrapSampleTransform(Host, new BootstrapSampleTransform.Arguments(), Data.Data);
var dataTrain = RoleMappedData.Create(viewTrain, Data.Schema.GetColumnRoleNames());
yield return(FeatureSelector.SelectFeatures(dataTrain, rand));
}
}
public IEnumerable <Batch> GetBatches(IRandom rand)
{
Host.Assert(Data != null, "Must call Initialize first!");
Host.AssertValue(rand);
using (var ch = Host.Start("Getting batches"))
{
RoleMappedData dataTest;
RoleMappedData dataTrain;
// Split the data, if needed.
if (!(ValidationDatasetProportion > 0))
{
dataTest = dataTrain = Data;
}
else
{
// Split the data into train and test sets.
string name = Data.Data.Schema.GetTempColumnName();
var args = new GenerateNumberTransform.Arguments();
args.Column = new[] { new GenerateNumberTransform.Column()
{
Name = name
} };
args.Seed = (uint)rand.Next();
var view = new GenerateNumberTransform(Host, args, Data.Data);
var viewTest = new RangeFilter(Host, new RangeFilter.Arguments()
{
Column = name, Max = ValidationDatasetProportion
}, view);
var viewTrain = new RangeFilter(Host, new RangeFilter.Arguments()
{
Column = name, Max = ValidationDatasetProportion, Complement = true
}, view);
dataTest = RoleMappedData.Create(viewTest, Data.Schema.GetColumnRoleNames());
dataTrain = RoleMappedData.Create(viewTrain, Data.Schema.GetColumnRoleNames());
}
if (BatchSize > 0)
{
// REVIEW: How should we carve the data into batches?
ch.Warning("Batch support is temporarily disabled");
}
yield return(new Batch(dataTrain, dataTest));
ch.Done();
}
}
private static TOut CreatePipelineEnsemble <TOut>(IHostEnvironment env, IPredictorModel[] predictors, SchemaBindablePipelineEnsembleBase ensemble)
where TOut : CommonOutputs.TrainerOutput, new()
{
var inputSchema = predictors[0].TransformModel.InputSchema;
var dv = new EmptyDataView(env, inputSchema);
// The role mappings are specific to the individual predictors.
var rmd = RoleMappedData.Create(dv);
var predictorModel = new PredictorModel(env, rmd, dv, ensemble);
var output = new TOut {
PredictorModel = predictorModel
};
return(output);
}
public override IEnumerable <Subset> GetSubsets(Batch batch, IRandom rand)
{
string name = Data.Data.Schema.GetTempColumnName();
var args = new GenerateNumberTransform.Arguments();
args.Column = new[] { new GenerateNumberTransform.Column()
{
Name = name
} };
args.Seed = (uint)rand.Next();
IDataTransform view = new GenerateNumberTransform(Host, args, Data.Data);
// REVIEW: This won't be very efficient when Size is large.
for (int i = 0; i < Size; i++)
{
var viewTrain = new RangeFilter(Host, new RangeFilter.Arguments()
{
Column = name, Min = (Double)i / Size, Max = (Double)(i + 1) / Size
}, view);
var dataTrain = RoleMappedData.Create(viewTrain, Data.Schema.GetColumnRoleNames());
yield return(FeatureSelector.SelectFeatures(dataTrain, rand));
}
}
public static CombinedOutput CombineMetrics(IHostEnvironment env, CombineMetricsInput input)
{
var eval = GetEvaluator(env, input.Kind);
var perInst = EvaluateUtils.ConcatenatePerInstanceDataViews(env, eval, true, true, input.PerInstanceMetrics.Select(
idv => RoleMappedData.Create(idv, RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, input.LabelColumn))).ToArray(),
out var variableSizeVectorColumnNames);
var warnings = input.Warnings != null ? new List <IDataView>(input.Warnings) : new List <IDataView>();
if (variableSizeVectorColumnNames.Length > 0)
{
var dvBldr = new ArrayDataViewBuilder(env);
var warn = $"Detected columns of variable length: {string.Join(", ", variableSizeVectorColumnNames)}." +
$" Consider setting collateMetrics- for meaningful per-Folds results.";
dvBldr.AddColumn(MetricKinds.ColumnNames.WarningText, TextType.Instance, new DvText(warn));
warnings.Add(dvBldr.GetDataView());
}
env.Assert(Utils.Size(perInst) == 1);
var overall = eval.GetOverallResults(input.OverallMetrics);
overall = EvaluateUtils.CombineFoldMetricsDataViews(env, overall, input.OverallMetrics.Length);
IDataView conf = null;
if (Utils.Size(input.ConfusionMatrix) > 0)
{
EvaluateUtils.ReconcileSlotNames <double>(env, input.ConfusionMatrix, MetricKinds.ColumnNames.Count, NumberType.R8);
for (int i = 0; i < input.ConfusionMatrix.Length; i++)
{
var idv = input.ConfusionMatrix[i];
// Find the old Count column and drop it.
for (int col = 0; col < idv.Schema.ColumnCount; col++)
{
if (idv.Schema.IsHidden(col) &&
idv.Schema.GetColumnName(col).Equals(MetricKinds.ColumnNames.Count))
{
input.ConfusionMatrix[i] = new ChooseColumnsByIndexTransform(env,
new ChooseColumnsByIndexTransform.Arguments()
{
Drop = true, Index = new[] { col }
}, idv);
break;
}
}
}
conf = EvaluateUtils.ConcatenateOverallMetrics(env, input.ConfusionMatrix);
}
var warningsIdv = warnings.Count > 0 ? AppendRowsDataView.Create(env, warnings[0].Schema, warnings.ToArray()) : null;
return(new CombinedOutput()
{
PerInstanceMetrics = perInst[0],
OverallMetrics = overall,
ConfusionMatrix = conf,
Warnings = warningsIdv
});
}
private void Run(IChannel ch)
{
IDataLoader loader = null;
IPredictor rawPred = null;
IDataView view;
RoleMappedSchema trainSchema = null;
if (_model == null)
{
if (string.IsNullOrEmpty(Args.InputModelFile))
{
loader = CreateLoader();
rawPred = null;
trainSchema = null;
Host.CheckUserArg(Args.LoadPredictor != true, nameof(Args.LoadPredictor),
"Cannot be set to true unless " + nameof(Args.InputModelFile) + " is also specifified.");
}
else
{
LoadModelObjects(ch, _loadPredictor, out rawPred, true, out trainSchema, out loader);
}
view = loader;
}
else
{
view = _model.Apply(Host, new EmptyDataView(Host, _model.InputSchema));
}
// Get the transform chain.
IDataView source;
IDataView end;
LinkedList <ITransformCanSaveOnnx> transforms;
GetPipe(ch, view, out source, out end, out transforms);
Host.Assert(transforms.Count == 0 || transforms.Last.Value == end);
var assembly = System.Reflection.Assembly.GetExecutingAssembly();
var versionInfo = System.Diagnostics.FileVersionInfo.GetVersionInfo(assembly.Location);
var ctx = new OnnxContextImpl(Host, _name, ProducerName, versionInfo.FileVersion,
ModelVersion, _domain);
// If we have a predictor, try to get the scorer for it.
if (rawPred != null)
{
RoleMappedData data;
if (trainSchema != null)
{
data = RoleMappedData.Create(end, trainSchema.GetColumnRoleNames());
}
else
{
// We had a predictor, but no roles stored in the model. Just suppose
// default column names are OK, if present.
data = TrainUtils.CreateExamplesOpt(end, DefaultColumnNames.Label,
DefaultColumnNames.Features, DefaultColumnNames.GroupId, DefaultColumnNames.Weight, DefaultColumnNames.Name);
}
var scorePipe = ScoreUtils.GetScorer(rawPred, data, Host, trainSchema);
var scoreOnnx = scorePipe as ITransformCanSaveOnnx;
if (scoreOnnx?.CanSaveOnnx == true)
{
Host.Assert(scorePipe.Source == end);
end = scorePipe;
transforms.AddLast(scoreOnnx);
}
else
{
Contracts.CheckUserArg(_loadPredictor != true,
nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but we do not know how to save it as ONNX.");
ch.Warning("We do not know how to save the predictor as ONNX. Ignoring.");
}
}
else
{
Contracts.CheckUserArg(_loadPredictor != true,
nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but one was not present.");
}
HashSet <string> inputColumns = new HashSet <string>();
//Create graph inputs.
for (int i = 0; i < source.Schema.ColumnCount; i++)
{
string colName = source.Schema.GetColumnName(i);
if (_inputsToDrop.Contains(colName))
{
continue;
}
ctx.AddInputVariable(source.Schema.GetColumnType(i), colName);
inputColumns.Add(colName);
}
//Create graph nodes, outputs and intermediate values.
foreach (var trans in transforms)
{
Host.Assert(trans.CanSaveOnnx);
trans.SaveAsOnnx(ctx);
}
//Add graph outputs.
for (int i = 0; i < end.Schema.ColumnCount; ++i)
{
if (end.Schema.IsHidden(i))
{
continue;
}
var idataviewColumnName = end.Schema.GetColumnName(i);;
if (_outputsToDrop.Contains(idataviewColumnName) || _inputsToDrop.Contains(idataviewColumnName))
{
continue;
}
var variableName = ctx.TryGetVariableName(idataviewColumnName);
if (variableName != null)
{
ctx.AddOutputVariable(end.Schema.GetColumnType(i), variableName);
}
}
var model = ctx.MakeModel();
if (_outputModelPath != null)
{
using (var file = Host.CreateOutputFile(_outputModelPath))
using (var stream = file.CreateWriteStream())
model.WriteTo(stream);
}
if (_outputJsonModelPath != null)
{
using (var file = Host.CreateOutputFile(_outputJsonModelPath))
using (var stream = file.CreateWriteStream())
using (var writer = new StreamWriter(stream))
{
var parsedJson = JsonConvert.DeserializeObject(model.ToString());
writer.Write(JsonConvert.SerializeObject(parsedJson, Formatting.Indented));
}
}
if (!string.IsNullOrWhiteSpace(Args.OutputModelFile))
{
Contracts.Assert(loader != null);
ch.Trace("Saving the data pipe");
// Should probably include "end"?
SaveLoader(loader, Args.OutputModelFile);
}
}
public void Train(List <FeatureSubsetModel <IPredictorProducing <TOutput> > > models, RoleMappedData data, IHostEnvironment env)
{
Contracts.CheckValue(env, nameof(env));
var host = env.Register(Stacking.LoadName);
host.CheckValue(models, nameof(models));
host.CheckValue(data, nameof(data));
using (var ch = host.Start("Training stacked model"))
{
ch.Check(Meta == null, "Train called multiple times");
ch.Check(BasePredictorType != null);
var maps = new ValueMapper <VBuffer <Single>, TOutput> [models.Count];
for (int i = 0; i < maps.Length; i++)
{
Contracts.Assert(models[i].Predictor is IValueMapper);
var m = (IValueMapper)models[i].Predictor;
maps[i] = m.GetMapper <VBuffer <Single>, TOutput>();
}
// REVIEW: Should implement this better....
var labels = new Single[100];
var features = new VBuffer <Single> [100];
int count = 0;
// REVIEW: Should this include bad values or filter them?
using (var cursor = new FloatLabelCursor(data, CursOpt.AllFeatures | CursOpt.AllLabels))
{
TOutput[] predictions = new TOutput[maps.Length];
var vBuffers = new VBuffer <Single> [maps.Length];
while (cursor.MoveNext())
{
Parallel.For(0, maps.Length, i =>
{
var model = models[i];
if (model.SelectedFeatures != null)
{
EnsembleUtils.SelectFeatures(ref cursor.Features, model.SelectedFeatures, model.Cardinality, ref vBuffers[i]);
maps[i](ref vBuffers[i], ref predictions[i]);
}
else
{
maps[i](ref cursor.Features, ref predictions[i]);
}
});
Utils.EnsureSize(ref labels, count + 1);
Utils.EnsureSize(ref features, count + 1);
labels[count] = cursor.Label;
FillFeatureBuffer(predictions, ref features[count]);
count++;
}
}
ch.Info("The number of instances used for stacking trainer is {0}", count);
var bldr = new ArrayDataViewBuilder(host);
Array.Resize(ref labels, count);
Array.Resize(ref features, count);
bldr.AddColumn("Label", NumberType.Float, labels);
bldr.AddColumn("Features", NumberType.Float, features);
var view = bldr.GetDataView();
var rmd = RoleMappedData.Create(view, ColumnRole.Label.Bind("Label"), ColumnRole.Feature.Bind("Features"));
var trainer = BasePredictorType.CreateInstance(host);
if (trainer is ITrainerEx ex && ex.NeedNormalization)
{
ch.Warning("The trainer specified for stacking wants normalization, but we do not currently allow this.");
}
trainer.Train(rmd);
Meta = trainer.CreatePredictor();
CheckMeta();
ch.Done();
}
}
public static TOut Train <TArg, TOut>(IHost host, TArg input,
Func <ITrainer> createTrainer,
Func <string> getLabel = null,
Func <string> getWeight = null,
Func <string> getGroup = null,
Func <string> getName = null,
Func <IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > > getCustom = null,
ICalibratorTrainerFactory calibrator = null,
int maxCalibrationExamples = 0)
where TArg : LearnerInputBase
where TOut : CommonOutputs.TrainerOutput, new()
{
using (var ch = host.Start("Training"))
{
ISchema schema = input.TrainingData.Schema;
var feature = FindColumn(ch, schema, input.FeatureColumn);
var label = getLabel?.Invoke();
var weight = getWeight?.Invoke();
var group = getGroup?.Invoke();
var name = getName?.Invoke();
var custom = getCustom?.Invoke();
var trainer = createTrainer();
IDataView view = input.TrainingData;
TrainUtils.AddNormalizerIfNeeded(host, ch, trainer, ref view, feature, input.NormalizeFeatures);
ch.Trace("Binding columns");
var roleMappedData = TrainUtils.CreateExamples(view, label, feature, group, weight, name, custom);
RoleMappedData cachedRoleMappedData = roleMappedData;
Cache.CachingType?cachingType = null;
switch (input.Caching)
{
case CachingOptions.Memory:
{
cachingType = Cache.CachingType.Memory;
break;
}
case CachingOptions.Disk:
{
cachingType = Cache.CachingType.Disk;
break;
}
case CachingOptions.Auto:
{
ITrainerEx trainerEx = trainer as ITrainerEx;
// REVIEW: we should switch to hybrid caching in future.
if (!(input.TrainingData is BinaryLoader) && (trainerEx == null || trainerEx.WantCaching))
{
// default to Memory so mml is on par with maml
cachingType = Cache.CachingType.Memory;
}
break;
}
case CachingOptions.None:
break;
default:
throw ch.ExceptParam(nameof(input.Caching), "Unknown option for caching: '{0}'", input.Caching);
}
if (cachingType.HasValue)
{
var cacheView = Cache.CacheData(host, new Cache.CacheInput()
{
Data = roleMappedData.Data,
Caching = cachingType.Value
}).OutputData;
cachedRoleMappedData = RoleMappedData.Create(cacheView, roleMappedData.Schema.GetColumnRoleNames());
}
var predictor = TrainUtils.Train(host, ch, cachedRoleMappedData, trainer, "Train", calibrator, maxCalibrationExamples);
var output = new TOut()
{
PredictorModel = new PredictorModel(host, roleMappedData, input.TrainingData, predictor)
};
ch.Done();
return(output);
}
}
private void Run(IChannel ch)
{
IDataLoader loader;
IPredictor rawPred;
RoleMappedSchema trainSchema;
if (string.IsNullOrEmpty(Args.InputModelFile))
{
loader = CreateLoader();
rawPred = null;
trainSchema = null;
Host.CheckUserArg(Args.LoadPredictor != true, nameof(Args.LoadPredictor),
"Cannot be set to true unless " + nameof(Args.InputModelFile) + " is also specifified.");
}
else
{
LoadModelObjects(ch, _loadPredictor, out rawPred, true, out trainSchema, out loader);
}
// Get the transform chain.
IDataView source;
IDataView end;
LinkedList <ITransformCanSavePfa> transforms;
GetPipe(ch, loader, out source, out end, out transforms);
Host.Assert(transforms.Count == 0 || transforms.Last.Value == end);
// If we have a predictor, try to get the scorer for it.
if (rawPred != null)
{
RoleMappedData data;
if (trainSchema != null)
{
data = RoleMappedData.Create(end, trainSchema.GetColumnRoleNames());
}
else
{
// We had a predictor, but no roles stored in the model. Just suppose
// default column names are OK, if present.
data = TrainUtils.CreateExamplesOpt(end, DefaultColumnNames.Label,
DefaultColumnNames.Features, DefaultColumnNames.GroupId, DefaultColumnNames.Weight, DefaultColumnNames.Name);
}
var scorePipe = ScoreUtils.GetScorer(rawPred, data, Host, trainSchema);
var scorePfa = scorePipe as ITransformCanSavePfa;
if (scorePfa?.CanSavePfa == true)
{
Host.Assert(scorePipe.Source == end);
end = scorePipe;
transforms.AddLast(scorePfa);
}
else
{
Contracts.CheckUserArg(_loadPredictor != true,
nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but we do not know how to save it as PFA.");
ch.Warning("We do not know how to save the predictor as PFA. Ignoring.");
}
}
else
{
Contracts.CheckUserArg(_loadPredictor != true,
nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but one was not present.");
}
var ctx = new BoundPfaContext(Host, source.Schema, _inputsToDrop, allowSet: _allowSet);
foreach (var trans in transforms)
{
Host.Assert(trans.CanSavePfa);
trans.SaveAsPfa(ctx);
}
var toExport = new List <string>();
for (int i = 0; i < end.Schema.ColumnCount; ++i)
{
if (end.Schema.IsHidden(i))
{
continue;
}
var name = end.Schema.GetColumnName(i);
if (_outputsToDrop.Contains(name))
{
continue;
}
if (!ctx.IsInput(name) || _keepInput)
{
toExport.Add(name);
}
}
JObject pfaDoc = ctx.Finalize(end.Schema, toExport.ToArray());
if (_name != null)
{
pfaDoc["name"] = _name;
}
if (_outputModelPath == null)
{
ch.Info(MessageSensitivity.Schema, pfaDoc.ToString(_formatting));
}
else
{
using (var file = Host.CreateOutputFile(_outputModelPath))
using (var stream = file.CreateWriteStream())
using (var writer = new StreamWriter(stream))
writer.Write(pfaDoc.ToString(_formatting));
}
if (!string.IsNullOrWhiteSpace(Args.OutputModelFile))
{
ch.Trace("Saving the data pipe");
// Should probably include "end"?
SaveLoader(loader, Args.OutputModelFile);
}
}