public static CommonOutputs.MacroOutput <Output> TrainTest(
IHostEnvironment env,
Arguments input,
EntryPointNode node)
{
// Create default pipeline ID if one not given.
input.PipelineId = input.PipelineId ?? Guid.NewGuid().ToString("N");
// Parse the subgraph.
var subGraphRunContext = new RunContext(env);
var subGraphNodes = EntryPointNode.ValidateNodes(env, subGraphRunContext, input.Nodes, node.Catalog);
// Change the subgraph to use the training data as input.
var varName = input.Inputs.Data.VarName;
VariableBinding transformModelVarName = null;
if (input.TransformModel != null)
{
transformModelVarName = node.GetInputVariable(nameof(input.TransformModel));
}
if (!subGraphRunContext.TryGetVariable(varName, out var dataVariable))
{
throw env.Except($"Invalid variable name '{varName}'.");
}
var trainingVar = node.GetInputVariable(nameof(input.TrainingData));
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.RenameInputVariable(dataVariable.Name, trainingVar);
}
subGraphRunContext.RemoveVariable(dataVariable);
// Change the subgraph to use the model variable as output.
varName = input.Outputs.Model.VarName;
if (!subGraphRunContext.TryGetVariable(varName, out dataVariable))
{
throw env.Except($"Invalid variable name '{varName}'.");
}
string outputVarName = node.GetOutputVariableName(nameof(Output.PredictorModel));
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.RenameOutputVariable(dataVariable.Name, outputVarName);
}
subGraphRunContext.RemoveVariable(dataVariable);
// Move the variables from the subcontext to the main context.
node.Context.AddContextVariables(subGraphRunContext);
// Change all the subgraph nodes to use the main context.
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.SetContext(node.Context);
}
// Testing using test data set
var testingVar = node.GetInputVariable(nameof(input.TestingData));
var exp = new Experiment(env);
//combine the predictor model with any potential transfrom model passed from the outer graph
if (transformModelVarName != null && transformModelVarName.VariableName != null)
{
var modelCombine = new ML.Transforms.TwoHeterogeneousModelCombiner
{
TransformModel = { VarName = transformModelVarName.VariableName },
PredictorModel = { VarName = outputVarName }
};
var modelCombineOutput = exp.Add(modelCombine);
outputVarName = modelCombineOutput.PredictorModel.VarName;
}
// Add the scoring node for testing.
var scoreNode = new ML.Transforms.DatasetScorer
{
Data = { VarName = testingVar.ToJson() },
PredictorModel = { VarName = outputVarName }
};
var scoreNodeOutput = exp.Add(scoreNode);
subGraphNodes.AddRange(EntryPointNode.ValidateNodes(env, node.Context, exp.GetNodes(), node.Catalog));
// Do not double-add previous nodes.
exp.Reset();
// REVIEW: we need to extract the proper label column name here to pass to the evaluators.
// This is where you would add code to do it.
var settings = new MacroUtils.EvaluatorSettings
{
LabelColumn = DefaultColumnNames.Label
};
string outVariableName;
if (input.IncludeTrainingMetrics)
{
// Add the scoring node for training.
var scoreNodeTraining = new ML.Transforms.DatasetScorer
{
Data = { VarName = trainingVar.ToJson() },
PredictorModel = { VarName = outputVarName }
};
var scoreNodeTrainingOutput = exp.Add(scoreNodeTraining);
subGraphNodes.AddRange(EntryPointNode.ValidateNodes(env, node.Context, exp.GetNodes(), node.Catalog));
// Do not double-add previous nodes.
exp.Reset();
// Add the evaluator node for training.
var evalInputOutputTraining = MacroUtils.GetEvaluatorInputOutput(input.Kind, settings);
var evalNodeTraining = evalInputOutputTraining.Item1;
var evalOutputTraining = evalInputOutputTraining.Item2;
evalNodeTraining.Data.VarName = scoreNodeTrainingOutput.ScoredData.VarName;
if (node.OutputMap.TryGetValue(nameof(Output.TrainingWarnings), out outVariableName))
{
evalOutputTraining.Warnings.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingOverallMetrics), out outVariableName))
{
evalOutputTraining.OverallMetrics.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingPerInstanceMetrics), out outVariableName))
{
evalOutputTraining.PerInstanceMetrics.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingConfusionMatrix), out outVariableName) &&
evalOutputTraining is CommonOutputs.IClassificationEvaluatorOutput eoTraining)
{
eoTraining.ConfusionMatrix.VarName = outVariableName;
}
exp.Add(evalNodeTraining, evalOutputTraining);
subGraphNodes.AddRange(EntryPointNode.ValidateNodes(env, node.Context, exp.GetNodes(), node.Catalog));
}
// Do not double-add previous nodes.
exp.Reset();
// Add the evaluator node for testing.
var evalInputOutput = MacroUtils.GetEvaluatorInputOutput(input.Kind, settings);
var evalNode = evalInputOutput.Item1;
var evalOutput = evalInputOutput.Item2;
evalNode.Data.VarName = scoreNodeOutput.ScoredData.VarName;
if (node.OutputMap.TryGetValue(nameof(Output.Warnings), out outVariableName))
{
evalOutput.Warnings.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.OverallMetrics), out outVariableName))
{
evalOutput.OverallMetrics.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.PerInstanceMetrics), out outVariableName))
{
evalOutput.PerInstanceMetrics.VarName = outVariableName;
}
if (node.OutputMap.TryGetValue(nameof(Output.ConfusionMatrix), out outVariableName) &&
evalOutput is CommonOutputs.IClassificationEvaluatorOutput eo)
{
eo.ConfusionMatrix.VarName = outVariableName;
}
exp.Add(evalNode, evalOutput);
subGraphNodes.AddRange(EntryPointNode.ValidateNodes(env, node.Context, exp.GetNodes(), node.Catalog));
// Marks as an atomic unit that can be run in
// a distributed fashion.
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.StageId = input.PipelineId;
}
return(new CommonOutputs.MacroOutput <Output>()
{
Nodes = subGraphNodes
});
}
public static CommonOutputs.MacroOutput <Output> TrainTest(
IHostEnvironment env,
Arguments input,
EntryPointNode node)
{
// Create default pipeline ID if one not given.
input.PipelineId = input.PipelineId ?? Guid.NewGuid().ToString("N");
// Parse the subgraph.
var subGraphRunContext = new RunContext(env);
var subGraphNodes = EntryPointNode.ValidateNodes(env, subGraphRunContext, input.Nodes, label: input.LabelColumn,
input.GroupColumn.IsExplicit ? input.GroupColumn.Value : null,
input.WeightColumn.IsExplicit ? input.WeightColumn.Value : null,
input.NameColumn.IsExplicit ? input.NameColumn.Value : null);
// Change the subgraph to use the training data as input.
var varName = input.Inputs.Data.VarName;
VariableBinding transformModelVarName = null;
if (input.TransformModel != null)
{
transformModelVarName = node.GetInputVariable(nameof(input.TransformModel));
}
if (!subGraphRunContext.TryGetVariable(varName, out var dataVariable))
{
throw env.Except($"Invalid variable name '{varName}'.");
}
var trainingVar = node.GetInputVariable(nameof(input.TrainingData));
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.RenameInputVariable(dataVariable.Name, trainingVar);
}
subGraphRunContext.RemoveVariable(dataVariable);
// Change the subgraph to use the model variable as output.
varName = input.Outputs.PredictorModel.VarName;
if (!subGraphRunContext.TryGetVariable(varName, out dataVariable))
{
throw env.Except($"Invalid variable name '{varName}'.");
}
string predictorModelVarName = node.GetOutputVariableName(nameof(Output.PredictorModel));
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.RenameOutputVariable(dataVariable.Name, predictorModelVarName);
}
subGraphRunContext.RemoveVariable(dataVariable);
// Move the variables from the subcontext to the main context.
node.Context.AddContextVariables(subGraphRunContext);
// Change all the subgraph nodes to use the main context.
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.SetContext(node.Context);
}
// Testing using test data set
var testingVar = node.GetInputVariable(nameof(input.TestingData));
//var exp = new Experiment(env);
Dictionary <string, List <ParameterBinding> > inputBindingMap;
Dictionary <ParameterBinding, VariableBinding> inputMap;
ParameterBinding paramBinding;
Dictionary <string, string> outputMap;
//combine the predictor model with any potential transfrom model passed from the outer graph
if (transformModelVarName != null && transformModelVarName.VariableName != null)
{
var combineArgs = new ModelOperations.SimplePredictorModelInput();
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var inputTransformModel = new SimpleVariableBinding(transformModelVarName.VariableName);
var inputPredictorModel = new SimpleVariableBinding(predictorModelVarName);
paramBinding = new SimpleParameterBinding(nameof(combineArgs.TransformModel));
inputBindingMap.Add(nameof(combineArgs.TransformModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, inputTransformModel);
paramBinding = new SimpleParameterBinding(nameof(combineArgs.PredictorModel));
inputBindingMap.Add(nameof(combineArgs.PredictorModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, inputPredictorModel);
outputMap = new Dictionary <string, string>();
var combineNodeOutputPredictorModel = new Var <PredictorModel>();
predictorModelVarName = combineNodeOutputPredictorModel.VarName;
outputMap.Add(nameof(ModelOperations.PredictorModelOutput.PredictorModel), combineNodeOutputPredictorModel.VarName);
EntryPointNode combineNode = EntryPointNode.Create(env, "Transforms.TwoHeterogeneousModelCombiner", combineArgs,
node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(combineNode);
}
// Add the scoring node for testing.
var args = new ScoreModel.Input();
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
paramBinding = new SimpleParameterBinding(nameof(args.Data));
inputBindingMap.Add(nameof(args.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, testingVar);
var scoreNodeInputPredictorModel = new SimpleVariableBinding(predictorModelVarName);
paramBinding = new SimpleParameterBinding(nameof(args.PredictorModel));
inputBindingMap.Add(nameof(args.PredictorModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, scoreNodeInputPredictorModel);
var scoreNodeOutputScoredData = new Var <IDataView>();
var scoreNodeOutputScoringTransform = new Var <TransformModel>();
outputMap = new Dictionary <string, string>();
outputMap.Add(nameof(ScoreModel.Output.ScoredData), scoreNodeOutputScoredData.VarName);
outputMap.Add(nameof(ScoreModel.Output.ScoringTransform), scoreNodeOutputScoringTransform.VarName);
EntryPointNode scoreNode = EntryPointNode.Create(env, "Transforms.DatasetScorer", args,
node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(scoreNode);
var evalDataVarName = scoreNodeOutputScoredData.VarName;
// REVIEW: add similar support for FeatureColumn.
var settings = new MacroUtils.EvaluatorSettings
{
LabelColumn = input.LabelColumn,
WeightColumn = input.WeightColumn.IsExplicit ? input.WeightColumn.Value : null,
GroupColumn = input.GroupColumn.IsExplicit ? input.GroupColumn.Value : null,
NameColumn = input.NameColumn.IsExplicit ? input.NameColumn.Value : null
};
if (input.IncludeTrainingMetrics)
{
string evalTrainingDataVarName;
args = new ScoreModel.Input();
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
paramBinding = new SimpleParameterBinding(nameof(args.Data));
inputBindingMap.Add(nameof(args.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, trainingVar);
scoreNodeInputPredictorModel = new SimpleVariableBinding(predictorModelVarName);
paramBinding = new SimpleParameterBinding(nameof(args.PredictorModel));
inputBindingMap.Add(nameof(args.PredictorModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, scoreNodeInputPredictorModel);
scoreNodeOutputScoredData = new Var <IDataView>();
scoreNodeOutputScoringTransform = new Var <TransformModel>();
outputMap = new Dictionary <string, string>();
outputMap.Add(nameof(ScoreModel.Output.ScoredData), scoreNodeOutputScoredData.VarName);
outputMap.Add(nameof(ScoreModel.Output.ScoringTransform), scoreNodeOutputScoringTransform.VarName);
scoreNode = EntryPointNode.Create(env, "Transforms.DatasetScorer", args,
node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(scoreNode);
evalTrainingDataVarName = scoreNodeOutputScoredData.VarName;
// Add the evaluator node for training.
var evalTrainingArgs = MacroUtils.GetEvaluatorArgs(input.Kind, out var evalTrainingEntryPointName, settings);
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var evalTrainingNodeInputData = new SimpleVariableBinding(evalTrainingDataVarName);
paramBinding = new SimpleParameterBinding(nameof(evalTrainingArgs.Data));
inputBindingMap.Add(nameof(evalTrainingArgs.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, evalTrainingNodeInputData);
outputMap = new Dictionary <string, string>();
if (node.OutputMap.TryGetValue(nameof(Output.TrainingWarnings), out var outTrainingVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.Warnings), outTrainingVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingOverallMetrics), out outTrainingVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.OverallMetrics), outTrainingVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingPerInstanceMetrics), out outTrainingVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.PerInstanceMetrics), outTrainingVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.TrainingConfusionMatrix), out outTrainingVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.ConfusionMatrix), outTrainingVariableName);
}
EntryPointNode evalTrainingNode = EntryPointNode.Create(env, evalTrainingEntryPointName, evalTrainingArgs, node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(evalTrainingNode);
}
// Add the evaluator node for testing.
var evalArgs = MacroUtils.GetEvaluatorArgs(input.Kind, out var evalEntryPointName, settings);
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var evalNodeInputData = new SimpleVariableBinding(evalDataVarName);
paramBinding = new SimpleParameterBinding(nameof(evalArgs.Data));
inputBindingMap.Add(nameof(evalArgs.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, evalNodeInputData);
outputMap = new Dictionary <string, string>();
if (node.OutputMap.TryGetValue(nameof(Output.Warnings), out var outVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.Warnings), outVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.OverallMetrics), out outVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.OverallMetrics), outVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.PerInstanceMetrics), out outVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.PerInstanceMetrics), outVariableName);
}
if (node.OutputMap.TryGetValue(nameof(Output.ConfusionMatrix), out outVariableName))
{
outputMap.Add(nameof(CommonOutputs.ClassificationEvaluateOutput.ConfusionMatrix), outVariableName);
}
EntryPointNode evalNode = EntryPointNode.Create(env, evalEntryPointName, evalArgs, node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(evalNode);
// Marks as an atomic unit that can be run in
// a distributed fashion.
foreach (var subGraphNode in subGraphNodes)
{
subGraphNode.StageId = input.PipelineId;
}
return(new CommonOutputs.MacroOutput <Output>()
{
Nodes = subGraphNodes
});
}
