GeneralFunctionAnalyzer <TIn, TDelegateInput, TOutShape>( IHostEnvironment env, IChannel ch, TDelegateInput input, ReaderReconciler <TIn> baseReconciler, Func <TDelegateInput, TOutShape> mapper, out IEstimator <ITransformer> estimator, Func <PipelineColumn, string> inputNameFunction) { Contracts.CheckValue(mapper, nameof(mapper)); var method = mapper.Method; var output = mapper(input); KeyValuePair <string, PipelineColumn>[] outPairs = StaticPipeInternalUtils.GetNamesValues(output, method.ReturnParameter); // Map where the key depends on the set of things in the value. The value contains the yet unresolved dependencies. var keyDependsOn = new Dictionary <PipelineColumn, HashSet <PipelineColumn> >(); // Map where the set of things in the value depend on the key. var dependsOnKey = new Dictionary <PipelineColumn, HashSet <PipelineColumn> >(); // The set of columns detected with zero dependencies. var zeroDependencies = new List <PipelineColumn>(); // First we build up the two structures above, using a queue and visiting from the outputs up. var toVisit = new Queue <PipelineColumn>(outPairs.Select(p => p.Value)); while (toVisit.Count > 0) { var col = toVisit.Dequeue(); ch.CheckParam(col != null, nameof(mapper), "The delegate seems to have null columns returned somewhere in the pipe."); if (keyDependsOn.ContainsKey(col)) { continue; // Already visited. } var dependsOn = new HashSet <PipelineColumn>(); foreach (var dep in col.Dependencies ?? Enumerable.Empty <PipelineColumn>()) { dependsOn.Add(dep); if (!dependsOnKey.TryGetValue(dep, out var dependsOnDep)) { dependsOnKey[dep] = dependsOnDep = new HashSet <PipelineColumn>(); toVisit.Enqueue(dep); } dependsOnDep.Add(col); } keyDependsOn[col] = dependsOn; if (dependsOn.Count == 0) { zeroDependencies.Add(col); } } // Get the base input columns. var baseInputs = keyDependsOn.Select(p => p.Key).Where(col => col.ReconcilerObj == baseReconciler).ToArray(); // The columns that utilize the base reconciler should have no dependencies. This could only happen if // the caller of this function has introduced a situation whereby they are claiming they can reconcile // to a data-reader object but still have input data dependencies, which does not make sense and // indicates that there is a bug in that component code. Unfortunately we can only detect that condition, // not determine exactly how it arose, but we can still do so to indicate to the user that there is a // problem somewhere in the stack. ch.CheckParam(baseInputs.All(col => keyDependsOn[col].Count == 0), nameof(input), "Bug detected where column producing object was yielding columns with dependencies."); // This holds the mappings of columns to names and back. Note that while the same column could be used on // the *output*, e.g., you could hypothetically have `(a: r.Foo, b: r.Foo)`, we treat that as the last thing // that is done. var nameMap = new BidirectionalDictionary <string, PipelineColumn>(); // Check to see if we have any set of initial names. This is important in the case where we are mapping // in an input data view. foreach (var col in baseInputs) { string inputName = inputNameFunction(col); if (inputName != null) { ch.Assert(!nameMap.ContainsKey(col)); ch.Assert(!nameMap.ContainsKey(inputName)); nameMap[col] = inputName; ch.Trace($"Using input with name {inputName}."); } } estimator = null; var toCopy = new List <(string src, string dst)>(); int tempNum = 0; // For all outputs, get potential name collisions with used inputs. Resolve by assigning the input a temporary name. foreach (var p in outPairs) { // If the name for the output is already used by one of the inputs, and this output column does not // happen to have the same name, then we need to rename that input to keep it available. if (nameMap.TryGetValue(p.Key, out var inputCol) && p.Value != inputCol) { ch.Assert(baseInputs.Contains(inputCol)); string tempName = $"#Temp_{tempNum++}"; ch.Trace($"Input/output name collision: Renaming '{p.Key}' to '{tempName}'."); toCopy.Add((p.Key, tempName)); nameMap[tempName] = nameMap[p.Key]; ch.Assert(!nameMap.ContainsKey(p.Key)); } // If we already have a name for this output column, maybe it is used elsewhere. (This can happen when // the only thing done with an input is we rename it, or output it twice, or something like this.) In // this case it is most appropriate to delay renaming till after all other processing has been done in // that case. But otherwise we may as well just take the name. if (!nameMap.ContainsKey(p.Value)) { nameMap[p.Key] = p.Value; } } // If any renamings were necessary, create the CopyColumns estimator. if (toCopy.Count > 0) { estimator = new CopyColumnsEstimator(env, toCopy.ToArray()); } // First clear the inputs from zero-dependencies yet to be resolved. foreach (var col in baseInputs) { ch.Assert(zeroDependencies.Contains(col)); ch.Assert(col.ReconcilerObj == baseReconciler); zeroDependencies.Remove(col); // Make more efficient... if (!dependsOnKey.TryGetValue(col, out var depends)) { continue; } // If any of these base inputs do not have names because, for example, they do not directly appear // in the outputs and otherwise do not have names, assign them a name. if (!nameMap.ContainsKey(col)) { nameMap[col] = $"Temp_{tempNum++}"; } foreach (var depender in depends) { var dependencies = keyDependsOn[depender]; ch.Assert(dependencies.Contains(col)); dependencies.Remove(col); if (dependencies.Count == 0) { zeroDependencies.Add(depender); } } dependsOnKey.Remove(col); } // Call the reconciler to get the base reader estimator. var readerEstimator = baseReconciler.Reconcile(env, baseInputs, nameMap.AsOther(baseInputs)); ch.AssertValueOrNull(readerEstimator); // Next we iteratively find those columns with zero dependencies, "create" them, and if anything depends on // these add them to the collection of zero dependencies, etc. etc. while (zeroDependencies.Count > 0) { // All columns with the same reconciler can be transformed together. // Note that the following policy of just taking the first group is not optimal. So for example, we // could have three columns, (a, b, c). If we had the output (a.X(), b.X() c.Y().X()), then maybe we'd // reconcile a.X() and b.X() together, then reconcile c.Y(), then reconcile c.Y().X() alone. Whereas, we // could have reconciled c.Y() first, then reconciled a.X(), b.X(), and c.Y().X() together. var group = zeroDependencies.GroupBy(p => p.ReconcilerObj).First(); // Beyond that first group that *might* be a data reader reconciler, all subsequent operations will // be on where the data is already loaded and so accept data as an input, that is, they should produce // an estimator. If this is not the case something seriously wonky is going on, most probably that the // user tried to use a column from another source. If this is detected we can produce a sensible error // message to tell them not to do this. if (!(group.Key is EstimatorReconciler rec)) { throw ch.Except("Columns from multiple sources were detected. " + "Did the caller use a " + nameof(PipelineColumn) + " from another delegate?"); } PipelineColumn[] cols = group.ToArray(); // All dependencies should, by this time, have names. ch.Assert(cols.SelectMany(c => c.Dependencies).All(dep => nameMap.ContainsKey(dep))); foreach (var newCol in cols) { if (!nameMap.ContainsKey(newCol)) { nameMap[newCol] = $"#Temp_{tempNum++}"; } } var localInputNames = nameMap.AsOther(cols.SelectMany(c => c.Dependencies ?? Enumerable.Empty <PipelineColumn>())); var localOutputNames = nameMap.AsOther(cols); var usedNames = new HashSet <string>(nameMap.Keys1.Except(localOutputNames.Values)); var localEstimator = rec.Reconcile(env, cols, localInputNames, localOutputNames, usedNames); readerEstimator = readerEstimator?.Append(localEstimator); estimator = estimator?.Append(localEstimator) ?? localEstimator; foreach (var newCol in cols) { zeroDependencies.Remove(newCol); // Make more efficient!! // Finally, we find all columns that depend on this one. If this happened to be the last pending // dependency, then we add it to the list. if (dependsOnKey.TryGetValue(newCol, out var depends)) { foreach (var depender in depends) { var dependencies = keyDependsOn[depender]; Contracts.Assert(dependencies.Contains(newCol)); dependencies.Remove(newCol); if (dependencies.Count == 0) { zeroDependencies.Add(depender); } } dependsOnKey.Remove(newCol); } } } if (keyDependsOn.Any(p => p.Value.Count > 0)) { // This might happen if the user does something incredibly strange, like, say, take some prior // lambda, assign a column to a local variable, then re-use it downstream in a different lambdas. // The user would have to go to some extraorindary effort to do that, but nonetheless we want to // fail with a semi-sensible error message. throw ch.Except("There were some leftover columns with unresolved dependencies. " + "Did the caller use a " + nameof(PipelineColumn) + " from another delegate?"); } // Now do the final renaming, if any is necessary. toCopy.Clear(); foreach (var p in outPairs) { // TODO: Right now we just write stuff out. Once the copy-columns estimator is in place // we ought to do this for real. Contracts.Assert(nameMap.ContainsKey(p.Value)); string currentName = nameMap[p.Value]; if (currentName != p.Key) { ch.Trace($"Will copy '{currentName}' to '{p.Key}'"); toCopy.Add((currentName, p.Key)); } } // If any final renamings were necessary, insert the appropriate CopyColumns transform. if (toCopy.Count > 0) { var copyEstimator = new CopyColumnsEstimator(env, toCopy.ToArray()); if (estimator == null) { estimator = copyEstimator; } else { estimator = estimator.Append(copyEstimator); } } ch.Trace($"Exiting {nameof(ReaderEstimatorAnalyzerHelper)}"); return(readerEstimator); }
private void RunCore(IChannel ch) { Host.AssertValue(ch); ch.Trace("Creating loader"); LoadModelObjects(ch, true, out var predictor, true, out var trainSchema, out var loader); ch.AssertValue(predictor); ch.AssertValueOrNull(trainSchema); ch.AssertValue(loader); ch.Trace("Creating pipeline"); var scorer = Args.Scorer; ch.Assert(scorer == null || scorer is ICommandLineComponentFactory, "ScoreCommand should only be used from the command line."); var bindable = ScoreUtils.GetSchemaBindableMapper(Host, predictor, scorerFactorySettings: scorer as ICommandLineComponentFactory); ch.AssertValue(bindable); // REVIEW: We probably ought to prefer role mappings from the training schema. string feat = TrainUtils.MatchNameOrDefaultOrNull(ch, loader.Schema, nameof(Args.FeatureColumn), Args.FeatureColumn, DefaultColumnNames.Features); string group = TrainUtils.MatchNameOrDefaultOrNull(ch, loader.Schema, nameof(Args.GroupColumn), Args.GroupColumn, DefaultColumnNames.GroupId); var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn); var schema = new RoleMappedSchema(loader.Schema, label: null, feature: feat, group: group, custom: customCols, opt: true); var mapper = bindable.Bind(Host, schema); if (scorer == null) { scorer = ScoreUtils.GetScorerComponent(mapper); } loader = CompositeDataLoader.ApplyTransform(Host, loader, "Scorer", scorer.ToString(), (env, view) => scorer.CreateComponent(env, view, mapper, trainSchema)); loader = CompositeDataLoader.Create(Host, loader, Args.PostTransform); if (!string.IsNullOrWhiteSpace(Args.OutputModelFile)) { ch.Trace("Saving the data pipe"); SaveLoader(loader, Args.OutputModelFile); } ch.Trace("Creating saver"); IDataSaver writer; if (Args.Saver == null) { var ext = Path.GetExtension(Args.OutputDataFile); var isText = ext == ".txt" || ext == ".tlc"; if (isText) { writer = new TextSaver(Host, new TextSaver.Arguments()); } else { writer = new BinarySaver(Host, new BinarySaver.Arguments()); } } else { writer = Args.Saver.CreateComponent(Host); } ch.Assert(writer != null); var outputIsBinary = writer is BinaryWriter; bool outputAllColumns = Args.OutputAllColumns == true || (Args.OutputAllColumns == null && Utils.Size(Args.OutputColumn) == 0 && outputIsBinary); bool outputNamesAndLabels = Args.OutputAllColumns == true || Utils.Size(Args.OutputColumn) == 0; if (Args.OutputAllColumns == true && Utils.Size(Args.OutputColumn) != 0) { ch.Warning(nameof(Args.OutputAllColumns) + "=+ always writes all columns irrespective of " + nameof(Args.OutputColumn) + " specified."); } if (!outputAllColumns && Utils.Size(Args.OutputColumn) != 0) { foreach (var outCol in Args.OutputColumn) { if (!loader.Schema.TryGetColumnIndex(outCol, out int dummyColIndex)) { throw ch.ExceptUserArg(nameof(Arguments.OutputColumn), "Column '{0}' not found.", outCol); } } } uint maxScoreId = 0; if (!outputAllColumns) { maxScoreId = loader.Schema.GetMaxMetadataKind(out int colMax, MetadataUtils.Kinds.ScoreColumnSetId); } ch.Assert(outputAllColumns || maxScoreId > 0); // score set IDs are one-based var cols = new List <int>(); for (int i = 0; i < loader.Schema.ColumnCount; i++) { if (!Args.KeepHidden && loader.Schema.IsHidden(i)) { continue; } if (!(outputAllColumns || ShouldAddColumn(loader.Schema, i, maxScoreId, outputNamesAndLabels))) { continue; } var type = loader.Schema.GetColumnType(i); if (writer.IsColumnSavable(type)) { cols.Add(i); } else { ch.Warning("The column '{0}' will not be written as it has unsavable column type.", loader.Schema.GetColumnName(i)); } } ch.Check(cols.Count > 0, "No valid columns to save"); ch.Trace("Scoring and saving data"); using (var file = Host.CreateOutputFile(Args.OutputDataFile)) using (var stream = file.CreateWriteStream()) writer.SaveData(stream, loader, cols.ToArray()); }
private MatrixFactorizationPredictor TrainCore(IChannel ch, RoleMappedData data, RoleMappedData validData) { Host.AssertValue(ch); ch.AssertValue(data); ch.AssertValueOrNull(validData); ColumnInfo matrixColumnIndexColInfo; ColumnInfo matrixRowIndexColInfo; ColumnInfo validMatrixColumnIndexColInfo = null; ColumnInfo validMatrixRowIndexColInfo = null; ch.CheckValue(data.Schema.Label, nameof(data), "Input data did not have a unique label"); RecommenderUtils.CheckAndGetMatrixIndexColumns(data, out matrixColumnIndexColInfo, out matrixRowIndexColInfo, isDecode: false); if (data.Schema.Label.Type != NumberType.R4 && data.Schema.Label.Type != NumberType.R8) { throw ch.Except("Column '{0}' for label should be floating point, but is instead {1}", data.Schema.Label.Name, data.Schema.Label.Type); } MatrixFactorizationPredictor predictor; if (validData != null) { ch.CheckValue(validData, nameof(validData)); ch.CheckValue(validData.Schema.Label, nameof(validData), "Input validation data did not have a unique label"); RecommenderUtils.CheckAndGetMatrixIndexColumns(validData, out validMatrixColumnIndexColInfo, out validMatrixRowIndexColInfo, isDecode: false); if (validData.Schema.Label.Type != NumberType.R4 && validData.Schema.Label.Type != NumberType.R8) { throw ch.Except("Column '{0}' for validation label should be floating point, but is instead {1}", data.Schema.Label.Name, data.Schema.Label.Type); } if (!matrixColumnIndexColInfo.Type.Equals(validMatrixColumnIndexColInfo.Type)) { throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-column types differed, {0} vs. {1}", matrixColumnIndexColInfo.Type, validMatrixColumnIndexColInfo.Type); } if (!matrixRowIndexColInfo.Type.Equals(validMatrixRowIndexColInfo.Type)) { throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-row types differed, {0} vs. {1}", matrixRowIndexColInfo.Type, validMatrixRowIndexColInfo.Type); } } int colCount = matrixColumnIndexColInfo.Type.KeyCount; int rowCount = matrixRowIndexColInfo.Type.KeyCount; ch.Assert(rowCount > 0); ch.Assert(colCount > 0); // Checks for equality on the validation set ensure it is correct here. using (var cursor = data.Data.GetRowCursor(c => c == matrixColumnIndexColInfo.Index || c == matrixRowIndexColInfo.Index || c == data.Schema.Label.Index)) { // LibMF works only over single precision floats, but we want to be able to consume either. var labGetter = RowCursorUtils.GetGetterAs <float>(NumberType.R4, cursor, data.Schema.Label.Index); var matrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberType.U4, cursor, matrixColumnIndexColInfo.Index); var matrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberType.U4, cursor, matrixRowIndexColInfo.Index); if (validData == null) { // Have the trainer do its work. using (var buffer = PrepareBuffer()) { buffer.Train(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter); predictor = new MatrixFactorizationPredictor(Host, buffer, matrixColumnIndexColInfo.Type.AsKey, matrixRowIndexColInfo.Type.AsKey); } } else { using (var validCursor = validData.Data.GetRowCursor( c => c == validMatrixColumnIndexColInfo.Index || c == validMatrixRowIndexColInfo.Index || c == validData.Schema.Label.Index)) { ValueGetter <float> validLabelGetter = RowCursorUtils.GetGetterAs <float>(NumberType.R4, validCursor, validData.Schema.Label.Index); var validMatrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberType.U4, validCursor, validMatrixColumnIndexColInfo.Index); var validMatrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberType.U4, validCursor, validMatrixRowIndexColInfo.Index); // Have the trainer do its work. using (var buffer = PrepareBuffer()) { buffer.TrainWithValidation(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter, validCursor, validLabelGetter, validMatrixRowIndexGetter, validMatrixColumnIndexGetter); predictor = new MatrixFactorizationPredictor(Host, buffer, matrixColumnIndexColInfo.Type.AsKey, matrixRowIndexColInfo.Type.AsKey); } } } } return(predictor); }
private MatrixFactorizationModelParameters TrainCore(IChannel ch, RoleMappedData data, RoleMappedData validData = null) { _host.AssertValue(ch); ch.AssertValue(data); ch.AssertValueOrNull(validData); ch.CheckParam(data.Schema.Label.HasValue, nameof(data), "Input data did not have a unique label"); RecommenderUtils.CheckAndGetMatrixIndexColumns(data, out var matrixColumnIndexColInfo, out var matrixRowIndexColInfo, isDecode: false); var labelCol = data.Schema.Label.Value; if (labelCol.Type != NumberDataViewType.Single && labelCol.Type != NumberDataViewType.Double) { throw ch.Except("Column '{0}' for label should be floating point, but is instead {1}", labelCol.Name, labelCol.Type); } MatrixFactorizationModelParameters predictor; if (validData != null) { ch.CheckValue(validData, nameof(validData)); ch.CheckParam(validData.Schema.Label.HasValue, nameof(validData), "Input validation data did not have a unique label"); RecommenderUtils.CheckAndGetMatrixIndexColumns(validData, out var validMatrixColumnIndexColInfo, out var validMatrixRowIndexColInfo, isDecode: false); var validLabelCol = validData.Schema.Label.Value; if (validLabelCol.Type != NumberDataViewType.Single && validLabelCol.Type != NumberDataViewType.Double) { throw ch.Except("Column '{0}' for validation label should be floating point, but is instead {1}", validLabelCol.Name, validLabelCol.Type); } if (!matrixColumnIndexColInfo.Type.Equals(validMatrixColumnIndexColInfo.Type)) { throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-column types differed, {0} vs. {1}", matrixColumnIndexColInfo.Type, validMatrixColumnIndexColInfo.Type); } if (!matrixRowIndexColInfo.Type.Equals(validMatrixRowIndexColInfo.Type)) { throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-row types differed, {0} vs. {1}", matrixRowIndexColInfo.Type, validMatrixRowIndexColInfo.Type); } } int colCount = matrixColumnIndexColInfo.Type.GetKeyCountAsInt32(_host); int rowCount = matrixRowIndexColInfo.Type.GetKeyCountAsInt32(_host); ch.Assert(rowCount > 0); ch.Assert(colCount > 0); // Checks for equality on the validation set ensure it is correct here. using (var cursor = data.Data.GetRowCursor(matrixColumnIndexColInfo, matrixRowIndexColInfo, data.Schema.Label.Value)) { // LibMF works only over single precision floats, but we want to be able to consume either. var labGetter = RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, cursor, data.Schema.Label.Value.Index); var matrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, cursor, matrixColumnIndexColInfo.Index); var matrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, cursor, matrixRowIndexColInfo.Index); if (validData == null) { // Have the trainer do its work. using (var buffer = PrepareBuffer()) { buffer.Train(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter); predictor = new MatrixFactorizationModelParameters(_host, buffer, (KeyDataViewType)matrixColumnIndexColInfo.Type, (KeyDataViewType)matrixRowIndexColInfo.Type); } } else { RecommenderUtils.CheckAndGetMatrixIndexColumns(validData, out var validMatrixColumnIndexColInfo, out var validMatrixRowIndexColInfo, isDecode: false); using (var validCursor = validData.Data.GetRowCursor(matrixColumnIndexColInfo, matrixRowIndexColInfo, data.Schema.Label.Value)) { ValueGetter <float> validLabelGetter = RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, validCursor, validData.Schema.Label.Value.Index); var validMatrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, validCursor, validMatrixColumnIndexColInfo.Index); var validMatrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, validCursor, validMatrixRowIndexColInfo.Index); // Have the trainer do its work. using (var buffer = PrepareBuffer()) { buffer.TrainWithValidation(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter, validCursor, validLabelGetter, validMatrixRowIndexGetter, validMatrixColumnIndexGetter); predictor = new MatrixFactorizationModelParameters(_host, buffer, (KeyDataViewType)matrixColumnIndexColInfo.Type, (KeyDataViewType)matrixRowIndexColInfo.Type); } } } } return(predictor); }
private void RunCore(IChannel ch) { ch.Trace("Constructing data pipeline"); IDataLoader loader; IPredictor predictor; RoleMappedSchema trainSchema; LoadModelObjects(ch, true, out predictor, true, out trainSchema, out loader); ch.AssertValue(predictor); ch.AssertValueOrNull(trainSchema); ch.AssertValue(loader); ch.Trace("Binding columns"); ISchema schema = loader.Schema; string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.LabelColumn), Args.LabelColumn, DefaultColumnNames.Label); string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.FeatureColumn), Args.FeatureColumn, DefaultColumnNames.Features); string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.GroupColumn), Args.GroupColumn, DefaultColumnNames.GroupId); string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.WeightColumn), Args.WeightColumn, DefaultColumnNames.Weight); string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.NameColumn), Args.NameColumn, DefaultColumnNames.Name); var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn); // Score. ch.Trace("Scoring and evaluating"); ch.Assert(Args.Scorer == null || Args.Scorer is ICommandLineComponentFactory, "TestCommand should only be used from the command line."); IDataScorerTransform scorePipe = ScoreUtils.GetScorer(Args.Scorer, predictor, loader, features, group, customCols, Host, trainSchema); // Evaluate. var evalComp = Args.Evaluator; if (!evalComp.IsGood()) { evalComp = EvaluateUtils.GetEvaluatorType(ch, scorePipe.Schema); } var evaluator = evalComp.CreateInstance(Host); var data = new RoleMappedData(scorePipe, label, null, group, weight, name, customCols); var metrics = evaluator.Evaluate(data); MetricWriter.PrintWarnings(ch, metrics); evaluator.PrintFoldResults(ch, metrics); if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall)) { throw ch.Except("No overall metrics found"); } overall = evaluator.GetOverallResults(overall); MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, 1); evaluator.PrintAdditionalMetrics(ch, metrics); Dictionary <string, IDataView>[] metricValues = { metrics }; SendTelemetryMetric(metricValues); if (!string.IsNullOrWhiteSpace(Args.OutputDataFile)) { var perInst = evaluator.GetPerInstanceMetrics(data); var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols); var idv = evaluator.GetPerInstanceDataViewToSave(perInstData); MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv); } }