private void RunCore(IChannel ch)
{
ch.Trace("Constructing data pipeline");
IDataLoader loader;
IPredictor predictor;
RoleMappedSchema trainSchema;
LoadModelObjects(ch, true, out predictor, true, out trainSchema, out loader);
ch.AssertValue(predictor);
ch.AssertValueOrNull(trainSchema);
ch.AssertValue(loader);
ch.Trace("Binding columns");
var schema = loader.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.FeatureColumn),
Args.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
// Score.
ch.Trace("Scoring and evaluating");
ch.Assert(Args.Scorer == null || Args.Scorer is ICommandLineComponentFactory, "TestCommand should only be used from the command line.");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(Args.Scorer, predictor, loader, features, group, customCols, Host, trainSchema);
// Evaluate.
var evaluator = Args.Evaluator?.CreateComponent(Host) ??
EvaluateUtils.GetEvaluator(Host, scorePipe.Schema);
var data = new RoleMappedData(scorePipe, label, null, group, weight, name, customCols);
var metrics = evaluator.Evaluate(data);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(data);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
private void RunCore(IChannel ch)
{
ch.Trace("Constructing data pipeline");
IDataLoader loader;
IPredictor predictor;
RoleMappedSchema trainSchema;
LoadModelObjects(ch, true, out predictor, true, out trainSchema, out loader);
ch.AssertValue(predictor);
ch.AssertValueOrNull(trainSchema);
ch.AssertValue(loader);
ch.Trace("Binding columns");
ISchema schema = loader.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.FeatureColumn),
Args.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
// Score.
ch.Trace("Scoring and evaluating");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(Args.Scorer, predictor, loader, features, group, customCols, Host, trainSchema);
// Evaluate.
var evalComp = Args.Evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, scorePipe.Schema);
}
var evaluator = evalComp.CreateInstance(Host);
var data = TrainUtils.CreateExamples(scorePipe, label, null, group, weight, name, customCols);
var metrics = evaluator.Evaluate(data);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
evaluator.PrintOverallResults(ch, Args.SummaryFilename, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(data);
var perInstData = TrainUtils.CreateExamples(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
/// <summary>
/// Checks the column types of the evaluator's input columns. The base class implementation checks only the type
/// of the weight column, and all other columns should be checked by the deriving classes in <see cref="CheckCustomColumnTypesCore"/>.
/// </summary>
protected void CheckColumnTypes(RoleMappedSchema schema)
{
// Check the weight column type.
if (schema.Weight != null)
{
EvaluateUtils.CheckWeightType(Host, schema.Weight.Type);
}
CheckScoreAndLabelTypes(schema);
// Check the other column types.
CheckCustomColumnTypesCore(schema);
}
protected override IEnumerable <string> GetPerInstanceColumnsToSave(RoleMappedSchema schema)
{
Host.CheckValue(schema, nameof(schema));
Host.CheckValue(schema.Label, nameof(schema), "Data must contain a label column");
// The anomaly detection evaluator outputs the label and the score.
yield return(schema.Label.Name);
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, schema.Schema, ScoreCol, nameof(Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.AnomalyDetection);
yield return(scoreInfo.Name);
// No additional output columns.
}
public BinaryClassificationMetrics Evaluate(IDataView data, string labelColumn = DefaultColumnNames.Label,
string probabilityColumn = DefaultColumnNames.Probability)
{
var ci = EvaluateUtils.GetScoreColumnInfo(_env, data.Schema, null, DefaultColumnNames.Score, MetadataUtils.Const.ScoreColumnKind.BinaryClassification);
var map = new KeyValuePair <RoleMappedSchema.ColumnRole, string>[]
{
RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Probability, probabilityColumn),
RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, ci.Name)
};
var rmd = new RoleMappedData(data, labelColumn, DefaultColumnNames.Features, opt: true, custom: map);
var metricsDict = _evaluator.Evaluate(rmd);
return(BinaryClassificationMetrics.FromMetrics(_env, metricsDict["OverallMetrics"], metricsDict["ConfusionMatrix"]).Single());
}
private void RunCore(IChannel ch)
{
Host.AssertValue(ch);
ch.Trace("Creating loader");
IDataView view = CreateAndSaveLoader(
(env, source) => new IO.BinaryLoader(env, new IO.BinaryLoader.Arguments(), source));
ch.Trace("Binding columns");
ISchema schema = view.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
ch.Trace("Creating evaluator");
var evalComp = Args.Evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, view.Schema);
}
var evaluator = evalComp.CreateInstance(Host);
var data = new RoleMappedData(view, label, null, group, weight, name, customCols);
var metrics = evaluator.Evaluate(data);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(data);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
protected override IEnumerable<ResultPack> EvaluateArticleResults(TargetItem evalTarget, ConfigBase evalConfig, ISourceValues evalValues, Period evalPeriod, IPeriodProfile evalProfile, IEnumerable<ResultPair> evalResults)
{
if (InternalEvaluate == null)
{
return EvaluateUtils.DecoratedError(ARTICLE_DESCRIPTION_ERROR_FORMAT, EXCEPTION_RESULT_NONE_TEXT);
}
var sourceBuilder = new EvaluateSource.SourceBuilder(evalValues);
var resultBuilder = new EvaluateSource.ResultBuilder(evalTarget, evalResults);
var bundleValues = PrepareConceptValues<EvaluateSource>(sourceBuilder, resultBuilder);
if (bundleValues.IsFailure)
{
return EvaluateUtils.DecoratedError(ARTICLE_DESCRIPTION_ERROR_FORMAT, bundleValues.Error);
}
return InternalEvaluate(evalConfig, evalPeriod, evalProfile, bundleValues);
}
protected override IEnumerable <string> GetPerInstanceColumnsToSave(RoleMappedSchema schema)
{
Host.CheckValue(schema, nameof(schema));
Host.CheckParam(schema.Label != null, nameof(schema), "Schema must contain a label column");
// The quantile regression evaluator outputs the label and score columns.
yield return(schema.Label.Name);
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, schema.Schema, ScoreCol, nameof(Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.QuantileRegression);
yield return(scoreInfo.Name);
// Return the output columns.
yield return(RegressionPerInstanceEvaluator.L1);
yield return(RegressionPerInstanceEvaluator.L2);
}
private IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetColumnRoles(
RoleMappedSchema testSchema, DataViewSchema scoredSchema)
{
switch (PredictionKind)
{
case PredictionKind.BinaryClassification:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Value.Name));
var scoreCol = EvaluateUtils.GetScoreColumn(Host, scoredSchema, null, nameof(BinaryClassifierMamlEvaluator.ArgumentsBase.ScoreColumn),
AnnotationUtils.Const.ScoreColumnKind.BinaryClassification);
yield return(RoleMappedSchema.CreatePair(AnnotationUtils.Const.ScoreValueKind.Score, scoreCol.Name));
// Get the optional probability column.
var probCol = EvaluateUtils.GetOptAuxScoreColumn(Host, scoredSchema, null, nameof(BinaryClassifierMamlEvaluator.Arguments.ProbabilityColumn),
scoreCol.Index, AnnotationUtils.Const.ScoreValueKind.Probability, NumberDataViewType.Single.Equals);
if (probCol.HasValue)
{
yield return(RoleMappedSchema.CreatePair(AnnotationUtils.Const.ScoreValueKind.Probability, probCol.Value.Name));
}
yield break;
case PredictionKind.Regression:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Value.Name));
scoreCol = EvaluateUtils.GetScoreColumn(Host, scoredSchema, null, nameof(RegressionMamlEvaluator.Arguments.ScoreColumn),
AnnotationUtils.Const.ScoreColumnKind.Regression);
yield return(RoleMappedSchema.CreatePair(AnnotationUtils.Const.ScoreValueKind.Score, scoreCol.Name));
yield break;
case PredictionKind.MulticlassClassification:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Value.Name));
scoreCol = EvaluateUtils.GetScoreColumn(Host, scoredSchema, null, nameof(MulticlassClassificationMamlEvaluator.Arguments.ScoreColumn),
AnnotationUtils.Const.ScoreColumnKind.MulticlassClassification);
yield return(RoleMappedSchema.CreatePair(AnnotationUtils.Const.ScoreValueKind.Score, scoreCol.Name));
yield break;
default:
throw Host.Except("Unrecognized prediction kind '{0}'", PredictionKind);
}
}
private IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetColumnRoles(
RoleMappedSchema testSchema, ISchema scoredSchema)
{
switch (PredictionKind)
{
case PredictionKind.BinaryClassification:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Name));
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, scoredSchema, null, nameof(BinaryClassifierMamlEvaluator.ArgumentsBase.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.BinaryClassification);
yield return(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreInfo.Name));
// Get the optional probability column.
var probInfo = EvaluateUtils.GetOptAuxScoreColumnInfo(Host, scoredSchema, null, nameof(BinaryClassifierMamlEvaluator.Arguments.ProbabilityColumn),
scoreInfo.Index, MetadataUtils.Const.ScoreValueKind.Probability, t => t == NumberType.Float);
if (probInfo != null)
{
yield return(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Probability, probInfo.Name));
}
yield break;
case PredictionKind.Regression:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Name));
scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, scoredSchema, null, nameof(RegressionMamlEvaluator.Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.Regression);
yield return(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreInfo.Name));
yield break;
case PredictionKind.MultiClassClassification:
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, testSchema.Label.Name));
scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, scoredSchema, null, nameof(MultiClassMamlEvaluator.Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.MultiClassClassification);
yield return(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreInfo.Name));
yield break;
default:
throw Host.Except("Unrecognized prediction kind '{0}'", PredictionKind);
}
}
/// <summary>
/// All the input columns needed by an evaluator should be added here.
/// The base class ipmlementation gets the score column, the label column (if exists) and the weight column (if exists).
/// Override if additional columns are needed.
/// </summary>
protected virtual IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetInputColumnRolesCore(RoleMappedSchema schema)
{
// Get the score column information.
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, schema.Schema, ScoreCol, nameof(ArgumentsBase.ScoreColumn),
ScoreColumnKind);
yield return(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreInfo.Name));
// Get the label column information.
string lab = EvaluateUtils.GetColName(LabelCol, schema.Label, DefaultColumnNames.Label);
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, lab));
var weight = EvaluateUtils.GetColName(WeightCol, schema.Weight, null);
if (!string.IsNullOrEmpty(weight))
{
yield return(RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Weight, weight));
}
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
IEmployProfile conceptProfile = evalProfile.Employ();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
TSeconds[] hoursFullWeeks = new TSeconds[0];
TSeconds[] hoursRealWeeks = new TSeconds[0];
TSeconds[] hoursFullMonth = new TSeconds[0];
TSeconds[] hoursRealMonth = new TSeconds[0];
if (conceptValues.ScheduleType == WorkScheduleType.SCHEDULE_NORMALY_WEEK)
{
hoursFullWeeks = conceptProfile.TimesheetWeekSchedule(evalPeriod, conceptValues.ShiftLiable, 5);
hoursRealWeeks = conceptProfile.TimesheetWeekSchedule(evalPeriod, conceptValues.ShiftActual, 5);
hoursFullMonth = conceptProfile.TimesheetFullSchedule(evalPeriod, hoursFullWeeks);
hoursRealMonth = conceptProfile.TimesheetFullSchedule(evalPeriod, hoursRealWeeks);
}
else
{
return(EvaluateUtils.DecoratedErrors(CONCEPT_DESCRIPTION_ERROR_FORMAT, SCHEDULE_TYPE_NOTIMPLEMENTED_TEXT));
}
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddWorkWeeksFullScheduleValue(hoursFullWeeks);
conceptResult.AddWorkWeeksRealScheduleValue(hoursRealWeeks);
conceptResult.AddWorkMonthFullScheduleValue(hoursFullMonth);
conceptResult.AddWorkMonthRealScheduleValue(hoursRealMonth);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
private void RunCore(IChannel ch)
{
Host.AssertValue(ch);
ch.Trace("Creating loader");
IDataView view = CreateAndSaveLoader(IO.BinaryLoader.LoadName);
ch.Trace("Binding columns");
ISchema schema = view.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
ch.Trace("Creating evaluator");
var evalComp = Args.Evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, view.Schema);
}
var evaluator = evalComp.CreateInstance(Host);
var data = TrainUtils.CreateExamples(view, label, null, group, weight, name, customCols);
var metrics = evaluator.Evaluate(data);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
evaluator.PrintOverallResults(ch, Args.SummaryFilename, metrics);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(data);
var perInstData = TrainUtils.CreateExamples(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
IEmployProfile conceptProfile = evalProfile.Employ();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
ITaxingProfile conceptProfile = evalProfile.Taxing();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
TAmountDec incomeGeneralRelated = conceptProfile.TaxableGeneralIncomes(evalPeriod, conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.DeclaracyType, conceptValues.ResidencyType,
conceptValues.TaxableIncome, conceptValues.PartnerIncome, conceptValues.ExcludeIncome);
TAmountDec incomeGeneralExclude = conceptProfile.ExcludeGeneralIncomes(evalPeriod, conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.DeclaracyType, conceptValues.ResidencyType,
conceptValues.TaxableIncome, conceptValues.PartnerIncome, conceptValues.ExcludeIncome);
TAmountDec incomeLolevelRelated = conceptProfile.TaxableLolevelIncomes(evalPeriod, conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.DeclaracyType, conceptValues.ResidencyType,
conceptValues.TaxableIncome, conceptValues.PartnerIncome, conceptValues.ExcludeIncome);
TAmountDec incomeAgrWorkRelated = conceptProfile.TaxableAgrWorkIncomes(evalPeriod, conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.DeclaracyType, conceptValues.ResidencyType,
conceptValues.TaxableIncome, conceptValues.PartnerIncome, conceptValues.ExcludeIncome);
TAmountDec incomePartnerRelated = conceptProfile.TaxablePartnerIncomes(evalPeriod, conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.DeclaracyType, conceptValues.ResidencyType,
conceptValues.TaxableIncome, conceptValues.PartnerIncome, conceptValues.ExcludeIncome);
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddIncomeTaxGeneralValue(conceptValues.SummarizeType,
conceptValues.StatementType, conceptValues.ResidencyType,
incomeGeneralRelated, incomeGeneralExclude,
incomeLolevelRelated, incomeAgrWorkRelated, incomePartnerRelated);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
protected override IEnumerable <string> GetPerInstanceColumnsToSave(RoleMappedSchema schema)
{
Host.CheckValue(schema, nameof(schema));
Host.CheckParam(schema.Label != null, nameof(schema), "Schema must contain a label column");
// The multi output regression evaluator outputs the label and score column if requested by the user.
if (!_supressScoresAndLabels)
{
yield return(schema.Label.Name);
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, schema.Schema, ScoreCol, nameof(Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.MultiOutputRegression);
yield return(scoreInfo.Name);
}
// Return the output columns.
yield return(MultiOutputRegressionPerInstanceEvaluator.L1);
yield return(MultiOutputRegressionPerInstanceEvaluator.L2);
yield return(MultiOutputRegressionPerInstanceEvaluator.Dist);
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
ITaxingProfile conceptProfile = evalProfile.Taxing();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
IHealthProfile healthsProfile = evalProfile.Health();
if (healthsProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, HEALTHS_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
TAmountDec startedBasisAmount = conceptProfile.TaxableIncomesAdvanceTaxingMode(evalPeriod,
conceptValues.GeneralIncome, conceptValues.ExcludeIncome,
conceptValues.LolevelIncome, conceptValues.TaskAgrIncome, conceptValues.PartnerIncome);
TAmountDec compoundPercFactor = healthsProfile.FactorCompound();
TAmountDec roundedBasisAmount = conceptProfile.DecRoundUp(startedBasisAmount);
TAmountDec cutdownBasisAmount = conceptProfile.TaxablePartialAdvanceHealth(evalPeriod, roundedBasisAmount, conceptValues.ExcludeIncome);
TAmountDec cutdownAboveAmount = conceptProfile.CutDownPartialAdvanceHealth(evalPeriod, roundedBasisAmount, conceptValues.ExcludeIncome);
TAmountDec finaledBasisAmount = conceptProfile.EployerPartialAdvanceHealth(evalPeriod, cutdownBasisAmount, compoundPercFactor);
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddMoneyInsuranceBasisValue(startedBasisAmount, roundedBasisAmount,
cutdownBasisAmount, cutdownAboveAmount, finaledBasisAmount);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
ITaxingProfile conceptProfile = evalProfile.Taxing();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddDeclarationTaxingValue(conceptValues.StatementType, conceptValues.SummarizeType,
conceptValues.DeclaracyType, conceptValues.ResidencyType, conceptValues.HealthAnnuity, conceptValues.SocialAnnuity);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
protected override IEnumerable <string> GetPerInstanceColumnsToSave(RoleMappedSchema schema)
{
Host.CheckValue(schema, nameof(schema));
Host.CheckValue(schema.Label, nameof(schema), "Data must contain a label column");
Host.CheckValue(schema.Group, nameof(schema), "Data must contain a group column");
// The ranking evaluator outputs the label, group key and score columns.
yield return(schema.Group.Name);
yield return(schema.Label.Name);
var scoreInfo = EvaluateUtils.GetScoreColumnInfo(Host, schema.Schema, ScoreCol, nameof(Arguments.ScoreColumn),
MetadataUtils.Const.ScoreColumnKind.Ranking);
yield return(scoreInfo.Name);
// Return the output columns.
yield return(RankerPerInstanceTransform.Ndcg);
yield return(RankerPerInstanceTransform.Dcg);
yield return(RankerPerInstanceTransform.MaxDcg);
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
ITaxingProfile conceptProfile = evalProfile.Taxing();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
TAmountDec solidaryBasis = conceptProfile.BasisSolidaryRounded(conceptValues.GeneralBaseAmount);
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddTaxSolidaryBaseValue(solidaryBasis);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
IHealthProfile conceptProfile = evalProfile.Health();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
Byte foreignerType = 0;
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddDeclarationHealthValue(conceptValues.StatementType,
conceptValues.SummarizeType, conceptValues.TotalYearBase, foreignerType);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
using (var ch = env.Register("EvaluateTransform").Start("Create Transform"))
{
ch.Trace("Binding columns");
var schema = input.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
args.LabelColumn, DefaultColumnNames.Label);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
args.WeightColumn, DefaultColumnNames.Weight);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, args.CustomColumn);
ch.Trace("Creating evaluator");
IMamlEvaluator eval = args.Evaluator?.CreateComponent(env) ??
EvaluateUtils.GetEvaluator(env, input.Schema);
var data = new RoleMappedData(input, label, null, group, weight, null, customCols);
return(eval.GetPerInstanceMetrics(data));
}
}
protected override IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetInputColumnRolesCore(RoleMappedSchema schema)
{
foreach (var col in base.GetInputColumnRolesCore(schema))
{
if (!col.Key.Equals(RoleMappedSchema.ColumnRole.Label))
{
yield return(col);
}
else if (schema.Schema.TryGetColumnIndex(col.Value, out int labelIndex))
{
yield return(col);
}
}
if (_calculateDbi)
{
string feat = EvaluateUtils.GetColName(_featureCol, schema.Feature, DefaultColumnNames.Features);
if (!schema.Schema.TryGetColumnIndex(feat, out int featCol))
{
throw Host.ExceptUserArg(nameof(Arguments.FeatureColumn), "Features column '{0}' not found", feat);
}
yield return(RoleMappedSchema.ColumnRole.Feature.Bind(feat));
}
}
public static IEnumerable <ResultPack> EvaluateConcept(ConfigBase evalConfig, Period evalPeriod, IPeriodProfile evalProfile,
Result <MasterItem.EvaluateSource, string> prepValues)
{
IEmployProfile conceptProfile = evalProfile.Employ();
if (conceptProfile == null)
{
return(EvaluateUtils.DecoratedError(CONCEPT_DESCRIPTION_ERROR_FORMAT, CONCEPT_PROFILE_NULL_TEXT));
}
MasterItem.EvaluateSource conceptValues = prepValues.Value;
// EVALUATION
TSeconds[] scheduleWorked = PeriodUtils.ScheduleBaseSubtract(
conceptValues.ScheduleMonth, conceptValues.AbsencesMonth, 1, 31);
// EVALUATION
IArticleResult conceptResult = new ArticleGeneralResult(evalConfig);
// SET RESULT VALUES
conceptResult.AddWorkMonthTermScheduleValue(scheduleWorked);
// SET RESULT VALUES
return(EvaluateUtils.Results(conceptResult));
}
public static CombinedOutput CombineMetrics(IHostEnvironment env, CombineMetricsInput input)
{
var eval = GetEvaluator(env, input.Kind);
var perInst = EvaluateUtils.ConcatenatePerInstanceDataViews(env, eval, true, true, input.PerInstanceMetrics.Select(
idv => RoleMappedData.CreateOpt(idv, new[]
{
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, input.LabelColumn),
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Weight, input.WeightColumn.Value),
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Group, input.GroupColumn.Value)
})).ToArray(),
out var variableSizeVectorColumnNames);
var warnings = input.Warnings != null ? new List <IDataView>(input.Warnings) : new List <IDataView>();
if (variableSizeVectorColumnNames.Length > 0)
{
var dvBldr = new ArrayDataViewBuilder(env);
var warn = $"Detected columns of variable length: {string.Join(", ", variableSizeVectorColumnNames)}." +
$" Consider setting collateMetrics- for meaningful per-Folds results.";
dvBldr.AddColumn(MetricKinds.ColumnNames.WarningText, TextType.Instance, new DvText(warn));
warnings.Add(dvBldr.GetDataView());
}
env.Assert(Utils.Size(perInst) == 1);
var overall = eval.GetOverallResults(input.OverallMetrics);
overall = EvaluateUtils.CombineFoldMetricsDataViews(env, overall, input.OverallMetrics.Length);
IDataView conf = null;
if (Utils.Size(input.ConfusionMatrix) > 0)
{
EvaluateUtils.ReconcileSlotNames <double>(env, input.ConfusionMatrix, MetricKinds.ColumnNames.Count, NumberType.R8);
for (int i = 0; i < input.ConfusionMatrix.Length; i++)
{
var idv = input.ConfusionMatrix[i];
// Find the old Count column and drop it.
for (int col = 0; col < idv.Schema.ColumnCount; col++)
{
if (idv.Schema.IsHidden(col) &&
idv.Schema.GetColumnName(col).Equals(MetricKinds.ColumnNames.Count))
{
input.ConfusionMatrix[i] = new ChooseColumnsByIndexTransform(env,
new ChooseColumnsByIndexTransform.Arguments()
{
Drop = true, Index = new[] { col }
}, idv);
break;
}
}
}
conf = EvaluateUtils.ConcatenateOverallMetrics(env, input.ConfusionMatrix);
}
var warningsIdv = warnings.Count > 0 ? AppendRowsDataView.Create(env, warnings[0].Schema, warnings.ToArray()) : null;
return(new CombinedOutput()
{
PerInstanceMetrics = perInst[0],
OverallMetrics = overall,
ConfusionMatrix = conf,
Warnings = warningsIdv
});
}
private FoldResult RunFold(int fold)
{
var host = GetHost();
host.Assert(0 <= fold && fold <= _numFolds);
// REVIEW: Make channels buffered in multi-threaded environments.
using (var ch = host.Start($"Fold {fold}"))
{
ch.Trace("Constructing trainer");
ITrainer trainer = _trainer.CreateInstance(host);
// Train pipe.
var trainFilter = new RangeFilter.Arguments();
trainFilter.Column = _splitColumn;
trainFilter.Min = (Double)fold / _numFolds;
trainFilter.Max = (Double)(fold + 1) / _numFolds;
trainFilter.Complement = true;
IDataView trainPipe = new RangeFilter(host, trainFilter, _inputDataView);
trainPipe = new OpaqueDataView(trainPipe);
var trainData = _createExamples(host, ch, trainPipe, trainer);
// Test pipe.
var testFilter = new RangeFilter.Arguments();
testFilter.Column = trainFilter.Column;
testFilter.Min = trainFilter.Min;
testFilter.Max = trainFilter.Max;
ch.Assert(!testFilter.Complement);
IDataView testPipe = new RangeFilter(host, testFilter, _inputDataView);
testPipe = new OpaqueDataView(testPipe);
var testData = _applyTransformsToTestData(host, ch, testPipe, trainData, trainPipe);
// Validation pipe and examples.
RoleMappedData validData = null;
if (_getValidationDataView != null)
{
ch.Assert(_applyTransformsToValidationData != null);
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validLoader = _getValidationDataView();
var validPipe = ApplyTransformUtils.ApplyAllTransformsToData(host, _inputDataView, validLoader);
validPipe = new OpaqueDataView(validPipe);
validData = _applyTransformsToValidationData(host, ch, validPipe, trainData, trainPipe);
}
}
// Train.
var predictor = TrainUtils.Train(host, ch, trainData, trainer, _trainer.Kind, validData,
_calibrator, _maxCalibrationExamples, _cacheData, _inputPredictor);
// Score.
ch.Trace("Scoring and evaluating");
var bindable = ScoreUtils.GetSchemaBindableMapper(host, predictor, _scorer);
ch.AssertValue(bindable);
var mapper = bindable.Bind(host, testData.Schema);
var scorerComp = _scorer.IsGood() ? _scorer : ScoreUtils.GetScorerComponent(mapper);
IDataScorerTransform scorePipe = scorerComp.CreateInstance(host, testData.Data, mapper, trainData.Schema);
// Save per-fold model.
string modelFileName = ConstructPerFoldName(_outputModelFile, fold);
if (modelFileName != null && _loader != null)
{
using (var file = host.CreateOutputFile(modelFileName))
{
var rmd = new RoleMappedData(
CompositeDataLoader.ApplyTransform(host, _loader, null, null,
(e, newSource) => ApplyTransformUtils.ApplyAllTransformsToData(e, trainData.Data, newSource)),
trainData.Schema.GetColumnRoleNames());
TrainUtils.SaveModel(host, ch, file, predictor, rmd, _cmd);
}
}
// Evaluate.
var evalComp = _evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, scorePipe.Schema);
}
var eval = evalComp.CreateInstance(host);
// Note that this doesn't require the provided columns to exist (because of the "opt" parameter).
// We don't normally expect the scorer to drop columns, but if it does, we should not require
// all the columns in the test pipeline to still be present.
var dataEval = new RoleMappedData(scorePipe, testData.Schema.GetColumnRoleNames(), opt: true);
var dict = eval.Evaluate(dataEval);
RoleMappedData perInstance = null;
if (_savePerInstance)
{
var perInst = eval.GetPerInstanceMetrics(dataEval);
perInstance = new RoleMappedData(perInst, dataEval.Schema.GetColumnRoleNames(), opt: true);
}
ch.Done();
return(new FoldResult(dict, dataEval.Schema.Schema, perInstance, trainData.Schema));
}
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
Host.AssertNonEmpty(cmd);
ch.Trace("Constructing trainer");
ITrainer trainer = Args.Trainer.CreateInstance(Host);
IPredictor inputPredictor = null;
if (Args.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, Args.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing the training pipeline");
IDataView trainPipe = CreateLoader();
ISchema schema = trainPipe.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.FeatureColumn),
Args.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
TrainUtils.AddNormalizerIfNeeded(Host, ch, trainer, ref trainPipe, features, Args.NormalizeFeatures);
ch.Trace("Binding columns");
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
var data = new RoleMappedData(trainPipe, label, features, group, weight, name, customCols);
RoleMappedData validData = null;
if (!string.IsNullOrWhiteSpace(Args.ValidationFile))
{
if (!TrainUtils.CanUseValidationData(trainer))
{
ch.Warning("Ignoring validationFile: Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validPipe = CreateRawLoader(dataFile: Args.ValidationFile);
validPipe = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, validPipe);
validData = new RoleMappedData(validPipe, data.Schema.GetColumnRoleNames());
}
}
var predictor = TrainUtils.Train(Host, ch, data, trainer, _info.LoadNames[0], validData,
Args.Calibrator, Args.MaxCalibrationExamples, Args.CacheData, inputPredictor);
IDataLoader testPipe;
using (var file = !string.IsNullOrEmpty(Args.OutputModelFile) ?
Host.CreateOutputFile(Args.OutputModelFile) : Host.CreateTempFile(".zip"))
{
TrainUtils.SaveModel(Host, ch, file, predictor, data, cmd);
ch.Trace("Constructing the testing pipeline");
using (var stream = file.OpenReadStream())
using (var rep = RepositoryReader.Open(stream, ch))
testPipe = LoadLoader(rep, Args.TestFile, true);
}
// Score.
ch.Trace("Scoring and evaluating");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(Args.Scorer, predictor, testPipe, features, group, customCols, Host, data.Schema);
// Evaluate.
var evalComp = Args.Evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, scorePipe.Schema);
}
var evaluator = evalComp.CreateInstance(Host);
var dataEval = new RoleMappedData(scorePipe, label, features,
group, weight, name, customCols, opt: true);
var metrics = evaluator.Evaluate(dataEval);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(dataEval);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
Host.AssertNonEmpty(cmd);
ch.Trace("Constructing trainer");
ITrainer trainer = ImplOptions.Trainer.CreateComponent(Host);
IPredictor inputPredictor = null;
if (ImplOptions.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, ImplOptions.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing the training pipeline");
IDataView trainPipe = CreateLoader();
var schema = trainPipe.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
ImplOptions.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.FeatureColumn),
ImplOptions.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
ImplOptions.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
ImplOptions.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
ImplOptions.NameColumn, DefaultColumnNames.Name);
TrainUtils.AddNormalizerIfNeeded(Host, ch, trainer, ref trainPipe, features, ImplOptions.NormalizeFeatures);
ch.Trace("Binding columns");
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, ImplOptions.CustomColumns);
var data = new RoleMappedData(trainPipe, label, features, group, weight, name, customCols);
RoleMappedData validData = null;
if (!string.IsNullOrWhiteSpace(ImplOptions.ValidationFile))
{
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Ignoring validationFile: Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validPipe = CreateRawLoader(dataFile: ImplOptions.ValidationFile);
validPipe = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, validPipe);
validData = new RoleMappedData(validPipe, data.Schema.GetColumnRoleNames());
}
}
// In addition to the training set, some trainers can accept two data sets, validation set and test set,
// in training phase. The major difference between validation set and test set is that training process may
// indirectly use validation set to improve the model but the learned model should totally independent of test set.
// Similar to validation set, the trainer can report the scores computed using test set.
RoleMappedData testDataUsedInTrainer = null;
if (!string.IsNullOrWhiteSpace(ImplOptions.TestFile))
{
// In contrast to the if-else block for validation above, we do not throw a warning if test file is provided
// because this is TrainTest command.
if (trainer.Info.SupportsTest)
{
ch.Trace("Constructing the test pipeline");
IDataView testPipeUsedInTrainer = CreateRawLoader(dataFile: ImplOptions.TestFile);
testPipeUsedInTrainer = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, testPipeUsedInTrainer);
testDataUsedInTrainer = new RoleMappedData(testPipeUsedInTrainer, data.Schema.GetColumnRoleNames());
}
}
var predictor = TrainUtils.Train(Host, ch, data, trainer, validData,
ImplOptions.Calibrator, ImplOptions.MaxCalibrationExamples, ImplOptions.CacheData, inputPredictor, testDataUsedInTrainer);
ILegacyDataLoader testPipe;
bool hasOutfile = !string.IsNullOrEmpty(ImplOptions.OutputModelFile);
var tempFilePath = hasOutfile ? null : Path.GetTempFileName();
using (var file = new SimpleFileHandle(ch, hasOutfile ? ImplOptions.OutputModelFile : tempFilePath, true, !hasOutfile))
{
TrainUtils.SaveModel(Host, ch, file, predictor, data, cmd);
ch.Trace("Constructing the testing pipeline");
using (var stream = file.OpenReadStream())
using (var rep = RepositoryReader.Open(stream, ch))
testPipe = LoadLoader(rep, ImplOptions.TestFile, true);
}
// Score.
ch.Trace("Scoring and evaluating");
ch.Assert(ImplOptions.Scorer == null || ImplOptions.Scorer is ICommandLineComponentFactory, "TrainTestCommand should only be used from the command line.");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(ImplOptions.Scorer, predictor, testPipe, features, group, customCols, Host, data.Schema);
// Evaluate.
var evaluator = ImplOptions.Evaluator?.CreateComponent(Host) ??
EvaluateUtils.GetEvaluator(Host, scorePipe.Schema);
var dataEval = new RoleMappedData(scorePipe, label, features,
group, weight, name, customCols, opt: true);
var metrics = evaluator.Evaluate(dataEval);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, ImplOptions.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(ImplOptions.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(dataEval);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, ImplOptions.OutputDataFile, idv);
}
}
protected virtual IDataView CombineOverallMetricsCore(IDataView[] metrics)
{
return(EvaluateUtils.ConcatenateOverallMetrics(Host, metrics));
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
IPredictor inputPredictor = null;
if (Args.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, Args.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing data pipeline");
IDataLoader loader = CreateRawLoader();
// If the per-instance results are requested and there is no name column, add a GenerateNumberTransform.
var preXf = Args.PreTransform;
if (!string.IsNullOrEmpty(Args.OutputDataFile))
{
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, loader.Schema, nameof(Args.NameColumn), Args.NameColumn, DefaultColumnNames.Name);
if (name == null)
{
var args = new GenerateNumberTransform.Arguments();
args.Column = new[] { new GenerateNumberTransform.Column()
{
Name = DefaultColumnNames.Name
}, };
args.UseCounter = true;
var options = CmdParser.GetSettings(ch, args, new GenerateNumberTransform.Arguments());
preXf = preXf.Concat(
new[]
{
new KeyValuePair <string, SubComponent <IDataTransform, SignatureDataTransform> >(
"", new SubComponent <IDataTransform, SignatureDataTransform>(
GenerateNumberTransform.LoadName, options))
}).ToArray();
}
}
loader = CompositeDataLoader.Create(Host, loader, preXf);
ch.Trace("Binding label and features columns");
IDataView pipe = loader;
var stratificationColumn = GetSplitColumn(ch, loader, ref pipe);
var scorer = Args.Scorer;
var evaluator = Args.Evaluator;
Func <IDataView> validDataCreator = null;
if (Args.ValidationFile != null)
{
validDataCreator =
() =>
{
// Fork the command.
var impl = new CrossValidationCommand(this);
return(impl.CreateRawLoader(dataFile: Args.ValidationFile));
};
}
FoldHelper fold = new FoldHelper(Host, RegistrationName, pipe, stratificationColumn,
Args, CreateRoleMappedData, ApplyAllTransformsToData, scorer, evaluator,
validDataCreator, ApplyAllTransformsToData, inputPredictor, cmd, loader, !string.IsNullOrEmpty(Args.OutputDataFile));
var tasks = fold.GetCrossValidationTasks();
if (!evaluator.IsGood())
{
evaluator = EvaluateUtils.GetEvaluatorType(ch, tasks[0].Result.ScoreSchema);
}
var eval = evaluator.CreateInstance(Host);
// Print confusion matrix and fold results for each fold.
for (int i = 0; i < tasks.Length; i++)
{
var dict = tasks[i].Result.Metrics;
MetricWriter.PrintWarnings(ch, dict);
eval.PrintFoldResults(ch, dict);
}
// Print the overall results.
if (!TryGetOverallMetrics(tasks.Select(t => t.Result.Metrics).ToArray(), out var overallList))
{
throw ch.Except("No overall metrics found");
}
var overall = eval.GetOverallResults(overallList.ToArray());
MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, Args.NumFolds);
eval.PrintAdditionalMetrics(ch, tasks.Select(t => t.Result.Metrics).ToArray());
Dictionary <string, IDataView>[] metricValues = tasks.Select(t => t.Result.Metrics).ToArray();
SendTelemetryMetric(metricValues);
// Save the per-instance results.
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInstance = EvaluateUtils.ConcatenatePerInstanceDataViews(Host, eval, Args.CollateMetrics,
Args.OutputExampleFoldIndex, tasks.Select(t => t.Result.PerInstanceResults).ToArray(), out var variableSizeVectorColumnNames);
if (variableSizeVectorColumnNames.Length > 0)
{
ch.Warning("Detected columns of variable length: {0}. Consider setting collateMetrics- for meaningful per-Folds results.",
string.Join(", ", variableSizeVectorColumnNames));
}
if (Args.CollateMetrics)
{
ch.Assert(perInstance.Length == 1);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, perInstance[0]);
}
else
{
int i = 0;
foreach (var idv in perInstance)
{
MetricWriter.SavePerInstance(Host, ch, ConstructPerFoldName(Args.OutputDataFile, i), idv);
i++;
}
}
}
}
