public abstract ISchemaBoundMapper Bind(IHostEnvironment env, RoleMappedSchema schema);
private protected override IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > GetInputColumnRolesCore(RoleMappedSchema schema) { foreach (var col in base.GetInputColumnRolesCore(schema)) { if (!col.Key.Equals(RoleMappedSchema.ColumnRole.Label)) { yield return(col); } else if (schema.Schema.TryGetColumnIndex(col.Value, out int labelIndex)) { yield return(col); } } if (_calculateDbi) { string feat = EvaluateUtils.GetColName(_featureCol, schema.Feature, DefaultColumnNames.Features); if (!schema.Schema.TryGetColumnIndex(feat, out int featCol)) { throw Host.ExceptUserArg(nameof(Arguments.FeatureColumn), "Features column '{0}' not found", feat); } yield return(RoleMappedSchema.ColumnRole.Feature.Bind(feat)); } }
public Bound(SchemaBindablePipelineEnsemble <T> parent, RoleMappedSchema schema) : base(parent, schema) { _combiner = parent.Combiner; }
public override ISchemaBoundMapper Bind(IHostEnvironment env, RoleMappedSchema schema) { return(new Bound(this, schema)); }
public override ISchemaBoundMapper Bind(IHostEnvironment env, RoleMappedSchema schema) { return(new XGBoostScalarRowMapper(schema, this, env, new ScoreMapperSchema(NumberType.Float, MetadataUtils.Const.ScoreColumnKind.Ranking))); }
/// <summary> /// Get the getter for the feature column, assuming it is a vector of float. /// </summary> public static ValueGetter <VBuffer <float> > GetFeatureFloatVectorGetter(this IRow row, RoleMappedSchema schema) { Contracts.CheckValue(row, nameof(row)); Contracts.CheckValue(schema, nameof(schema)); Contracts.CheckParam(schema.Schema == row.Schema, nameof(schema), "schemas don't match!"); Contracts.CheckParam(schema.Feature != null, nameof(schema), "Missing feature column"); return(row.GetGetter <VBuffer <float> >(schema.Feature.Index)); }
internal static IDataView GetSummaryAndStats(IHostEnvironment env, IPredictor predictor, RoleMappedSchema schema, out IDataView stats) { var calibrated = predictor as IWeaklyTypedCalibratedModelParameters; while (calibrated != null) { predictor = calibrated.WeaklyTypedSubModel; calibrated = predictor as IWeaklyTypedCalibratedModelParameters; } IDataView summary = null; stats = null; var dvGetter = predictor as ICanGetSummaryAsIDataView; var rowGetter = predictor as ICanGetSummaryAsIRow; if (dvGetter != null) { summary = dvGetter.GetSummaryDataView(schema); } if (rowGetter != null) { var row = rowGetter.GetSummaryIRowOrNull(schema); env.Check(dvGetter == null || row == null, "Predictor outputs two summary data views, don't know which one to choose"); if (row != null) { summary = RowCursorUtils.RowAsDataView(env, row); } var statsRow = rowGetter.GetStatsIRowOrNull(schema); if (statsRow != null) { stats = RowCursorUtils.RowAsDataView(env, statsRow); } } if (dvGetter == null && rowGetter == null) { var bldr = new ArrayDataViewBuilder(env); var summaryModel = predictor as ICanSaveSummary; // Save a data view containing one row and one column with the model summary. if (summaryModel != null) { var sb = new StringBuilder(); using (StringWriter sw = new StringWriter(sb)) summaryModel.SaveSummary(sw, schema); bldr.AddColumn("Summary", sb.ToString()); } else { bldr.AddColumn("PredictorName", predictor.GetType().ToString()); } summary = bldr.GetDataView(); } env.AssertValue(summary); return(summary); }
internal virtual DataViewSchema.Annotations MakeStatisticsMetadata(RoleMappedSchema schema, in VBuffer <ReadOnlyMemory <char> > names)
ISchemaBoundMapper ISchemaBindableMapper.Bind(IHostEnvironment env, RoleMappedSchema schema) => new FieldAwareFactorizationMachineScalarRowMapper(env, schema, Schema.Create(new BinaryClassifierSchema()), this);
public static IDataScorerTransform Create(IHostEnvironment env, TreeEnsembleFeaturizerBindableMapper.Arguments args, IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema) { return(new GenericScorer(env, args, data, mapper, trainSchema)); }
private protected abstract ISchemaBoundMapper BindCore(IHostEnvironment env, RoleMappedSchema schema);
ISchemaBoundMapper ISchemaBindableMapper.Bind(IHostEnvironment env, RoleMappedSchema schema) => BindCore(env, schema);
private protected override ISchemaBoundMapper BindCore(IHostEnvironment env, RoleMappedSchema schema) { return(new Bound(this, schema)); }
private void Run(IChannel ch) { IDataLoader loader = null; IPredictor rawPred = null; IDataView view; RoleMappedSchema trainSchema = null; if (_model == null) { if (string.IsNullOrEmpty(Args.InputModelFile)) { loader = CreateLoader(); rawPred = null; trainSchema = null; Host.CheckUserArg(Args.LoadPredictor != true, nameof(Args.LoadPredictor), "Cannot be set to true unless " + nameof(Args.InputModelFile) + " is also specifified."); } else { LoadModelObjects(ch, _loadPredictor, out rawPred, true, out trainSchema, out loader); } view = loader; } else { view = _model.Apply(Host, new EmptyDataView(Host, _model.InputSchema)); } // Create the ONNX context for storing global information var assembly = System.Reflection.Assembly.GetExecutingAssembly(); var versionInfo = System.Diagnostics.FileVersionInfo.GetVersionInfo(assembly.Location); var ctx = new OnnxContextImpl(Host, _name, ProducerName, versionInfo.FileVersion, ModelVersion, _domain, Args.OnnxVersion); // Get the transform chain. IDataView source; IDataView end; LinkedList <ITransformCanSaveOnnx> transforms; GetPipe(ctx, ch, view, out source, out end, out transforms); Host.Assert(transforms.Count == 0 || transforms.Last.Value == end); // If we have a predictor, try to get the scorer for it. if (rawPred != null) { RoleMappedData data; if (trainSchema != null) { data = new RoleMappedData(end, trainSchema.GetColumnRoleNames()); } else { // We had a predictor, but no roles stored in the model. Just suppose // default column names are OK, if present. data = new RoleMappedData(end, DefaultColumnNames.Label, DefaultColumnNames.Features, DefaultColumnNames.GroupId, DefaultColumnNames.Weight, DefaultColumnNames.Name, opt: true); } var scorePipe = ScoreUtils.GetScorer(rawPred, data, Host, trainSchema); var scoreOnnx = scorePipe as ITransformCanSaveOnnx; if (scoreOnnx?.CanSaveOnnx(ctx) == true) { Host.Assert(scorePipe.Source == end); end = scorePipe; transforms.AddLast(scoreOnnx); } else { Contracts.CheckUserArg(_loadPredictor != true, nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but we do not know how to save it as ONNX."); ch.Warning("We do not know how to save the predictor as ONNX. Ignoring."); } } else { Contracts.CheckUserArg(_loadPredictor != true, nameof(Arguments.LoadPredictor), "We were explicitly told to load the predictor but one was not present."); } HashSet <string> inputColumns = new HashSet <string>(); //Create graph inputs. for (int i = 0; i < source.Schema.ColumnCount; i++) { string colName = source.Schema.GetColumnName(i); if (_inputsToDrop.Contains(colName)) { continue; } ctx.AddInputVariable(source.Schema.GetColumnType(i), colName); inputColumns.Add(colName); } //Create graph nodes, outputs and intermediate values. foreach (var trans in transforms) { Host.Assert(trans.CanSaveOnnx(ctx)); trans.SaveAsOnnx(ctx); } //Add graph outputs. for (int i = 0; i < end.Schema.ColumnCount; ++i) { if (end.Schema.IsHidden(i)) { continue; } var idataviewColumnName = end.Schema.GetColumnName(i); // Since the last IDataView also contains columns of the initial IDataView, last IDataView's columns found in // _inputToDrop should be removed too. if (_inputsToDrop.Contains(idataviewColumnName) || _outputsToDrop.Contains(idataviewColumnName)) { continue; } var variableName = ctx.TryGetVariableName(idataviewColumnName); var trueVariableName = ctx.AddIntermediateVariable(null, idataviewColumnName, true); ctx.CreateNode("Identity", variableName, trueVariableName, ctx.GetNodeName("Identity"), ""); ctx.AddOutputVariable(end.Schema.GetColumnType(i), trueVariableName); } var model = ctx.MakeModel(); using (var file = Host.CreateOutputFile(_outputModelPath)) using (var stream = file.CreateWriteStream()) model.WriteTo(stream); if (_outputJsonModelPath != null) { using (var file = Host.CreateOutputFile(_outputJsonModelPath)) using (var stream = file.CreateWriteStream()) using (var writer = new StreamWriter(stream)) { var parsedJson = JsonConvert.DeserializeObject(model.ToString()); writer.Write(JsonConvert.SerializeObject(parsedJson, Formatting.Indented)); } } if (!string.IsNullOrWhiteSpace(Args.OutputModelFile)) { Contracts.Assert(loader != null); ch.Trace("Saving the data pipe"); // Should probably include "end"? SaveLoader(loader, Args.OutputModelFile); } }
private protected override IDataView GetPerInstanceMetricsCore(IDataView perInst, RoleMappedSchema schema) { // Wrap with a DropSlots transform to pick only the first _numTopClusters slots. if (perInst.Schema.TryGetColumnIndex(ClusteringPerInstanceEvaluator.SortedClusters, out int index)) { var type = perInst.Schema[index].Type; if (_numTopClusters < type.GetVectorSize()) { perInst = new SlotsDroppingTransformer(Host, ClusteringPerInstanceEvaluator.SortedClusters, min: _numTopClusters).Transform(perInst); } } if (perInst.Schema.TryGetColumnIndex(ClusteringPerInstanceEvaluator.SortedClusterScores, out index)) { var type = perInst.Schema[index].Type; if (_numTopClusters < type.GetVectorSize()) { perInst = new SlotsDroppingTransformer(Host, ClusteringPerInstanceEvaluator.SortedClusterScores, min: _numTopClusters).Transform(perInst); } } return(perInst); }