private MergedSchema(ISchema inputSchema, InternalSchemaDefinition addedSchema) : base(inputSchema) { Contracts.AssertValue(inputSchema); Contracts.AssertValue(addedSchema); Contracts.Assert(addedSchema.Columns.Length == 0); AddedSchema = addedSchema; }
/// <summary> /// Create a Cursorable object on a given data view. /// </summary> /// <param name="env">Host enviroment.</param> /// <param name="data">The underlying data view.</param> /// <param name="ignoreMissingColumns">Whether to ignore missing columns in the data view.</param> /// <param name="schemaDefinition">The optional user-provided schema.</param> /// <returns>The constructed Cursorable.</returns> public static TypedCursorable <TRow> Create(IHostEnvironment env, IDataView data, bool ignoreMissingColumns, SchemaDefinition schemaDefinition) { Contracts.AssertValue(env); env.AssertValue(data); env.AssertValueOrNull(schemaDefinition); var intSchemaDefn = InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition); return(new TypedCursorable <TRow>(env, data, ignoreMissingColumns, intSchemaDefn)); }
private MergedSchema(ISchema inputSchema, InternalSchemaDefinition addedSchema, string[] newColumnNames) : base(inputSchema, true, newColumnNames) { Contracts.AssertValue(inputSchema); Contracts.AssertValue(addedSchema); Contracts.AssertNonEmpty(newColumnNames); Contracts.Assert(addedSchema.Columns.Length == newColumnNames.Length); AddedSchema = addedSchema; }
public static InputRow <TRow> CreateInputRow <TRow>(IHostEnvironment env, SchemaDefinition schemaDefinition = null) where TRow : class { Contracts.AssertValue(env); env.AssertValueOrNull(schemaDefinition); var internalSchemaDefn = schemaDefinition == null ? InternalSchemaDefinition.Create(typeof(TRow), SchemaDefinition.Direction.Read) : InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition); return(new InputRow <TRow>(env, internalSchemaDefn)); }
/// <summary> /// The 'reapply' constructor. /// </summary> private StatefulFilterTransform(IHostEnvironment env, StatefulFilterTransform <TSrc, TDst, TState> transform, IDataView newSource) : base(env, RegistrationName, transform) { Host.AssertValue(transform); Host.AssertValue(newSource); _source = newSource; _filterFunc = transform._filterFunc; _typedSource = TypedCursorable <TSrc> .Create(Host, newSource, false, transform._inputSchemaDefinition); _addedSchema = transform._addedSchema; _bindings = new ColumnBindings(Schema.Create(newSource.Schema), DataViewConstructionUtils.GetSchemaColumns(_addedSchema)); }
public static MergedSchema Create(ISchema inputSchema, InternalSchemaDefinition addedSchema) { Contracts.AssertValue(inputSchema); Contracts.AssertValue(addedSchema); if (addedSchema.Columns.Length > 0) { var newColumnNames = addedSchema.Columns.Select(x => x.ColumnName).ToArray(); return(new MergedSchema(inputSchema, addedSchema, newColumnNames)); } return(new MergedSchema(inputSchema, addedSchema)); }
/// <summary> /// Returns whether the column type <paramref name="colType"/> can be bound to field <paramref name="memberInfo"/>. /// They must both be vectors or scalars, and the raw data kind should match. /// </summary> private static bool IsCompatibleType(ColumnType colType, MemberInfo memberInfo) { InternalSchemaDefinition.GetVectorAndKind(memberInfo, out bool isVector, out DataKind kind); if (isVector) { return(colType.IsVector && colType.ItemType.RawKind == kind); } else { return(!colType.IsVector && colType.RawKind == kind); } }
public static StreamingDataView <TRow> CreateFromEnumerable <TRow>(IHostEnvironment env, IEnumerable <TRow> data, SchemaDefinition schemaDefinition = null) where TRow : class { Contracts.AssertValue(env); env.AssertValue(data); env.AssertValueOrNull(schemaDefinition); var internalSchemaDefn = schemaDefinition == null ? InternalSchemaDefinition.Create(typeof(TRow), SchemaDefinition.Direction.Read) : InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition); return(new StreamingDataView <TRow>(env, data, internalSchemaDefn)); }
/// <summary> /// The 'reapply' constructor. /// </summary> private MapTransform(IHostEnvironment env, MapTransform <TSrc, TDst> transform, IDataView newSource) : base(env, RegistrationName, transform) { Host.AssertValue(transform); Host.AssertValue(newSource); Source = newSource; _mapAction = transform._mapAction; _typedSource = TypedCursorable <TSrc> .Create(Host, newSource, false, transform._inputSchemaDefinition); _addedSchema = transform._addedSchema; _bindings = new ColumnBindings(Data.Schema.Create(newSource.Schema), DataViewConstructionUtils.GetSchemaColumns(_addedSchema)); }
public static IDataView CreateFromList <TRow>(IHostEnvironment env, IList <TRow> data, SchemaDefinition schemaDefinition = null) where TRow : class { Contracts.AssertValue(env); env.AssertValue(data); env.AssertValueOrNull(schemaDefinition); var internalSchemaDefn = schemaDefinition == null ? InternalSchemaDefinition.Create(typeof(TRow)) : InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition); return(new ListDataView <TRow>(env, data, internalSchemaDefn)); }
private static Delegate[] MakePeeks(InternalSchemaDefinition schemaDef) { var peeks = new Delegate[schemaDef.Columns.Length]; for (var i = 0; i < peeks.Length; i++) { var currentColumn = schemaDef.Columns[i]; peeks[i] = currentColumn.IsComputed ? currentColumn.Generator : ApiUtils.GeneratePeek <InputRow <TRow>, TRow>(currentColumn); } return(peeks); }
/// <summary> /// Returns whether the column type <paramref name="colType"/> can be bound to field <paramref name="fieldInfo"/>. /// They must both be vectors or scalars, and the raw data kind should match. /// </summary> private static bool IsCompatibleType(ColumnType colType, FieldInfo fieldInfo) { bool isVector; DataKind kind; InternalSchemaDefinition.GetVectorAndKind(fieldInfo, out isVector, out kind); if (isVector) { return(colType.IsVector && colType.ItemType.RawKind == kind); } else { return(!colType.IsVector && colType.RawKind == kind); } }
protected DataViewBase(IHostEnvironment env, string name, InternalSchemaDefinition schemaDefn) { Contracts.AssertValue(env); env.AssertNonWhiteSpace(name); Host = env.Register(name); Host.AssertValue(schemaDefn); _schema = new SchemaProxy(schemaDefn); int n = _schema.SchemaDefn.Columns.Length; _peeks = new Delegate[n]; for (var i = 0; i < n; i++) { var currentColumn = _schema.SchemaDefn.Columns[i]; _peeks[i] = currentColumn.IsComputed ? currentColumn.Generator : ApiUtils.GeneratePeek <DataViewBase <TRow>, TRow>(currentColumn); } }
internal static Schema.DetachedColumn[] GetSchemaColumns(InternalSchemaDefinition schemaDefn) { Contracts.AssertValue(schemaDefn); var columns = new Schema.DetachedColumn[schemaDefn.Columns.Length]; for (int i = 0; i < columns.Length; i++) { var col = schemaDefn.Columns[i]; var meta = new MetadataBuilder(); foreach (var kvp in col.Metadata) { meta.Add(kvp.Value.Kind, kvp.Value.MetadataType, kvp.Value.GetGetterDelegate()); } columns[i] = new Schema.DetachedColumn(col.ColumnName, col.ColumnType, meta.GetMetadata()); } return(columns); }
public InputRowBase(IHostEnvironment env, Schema schema, InternalSchemaDefinition schemaDef, Delegate[] peeks, Func <int, bool> predicate) { Contracts.AssertValue(env); Host = env.Register("Row"); Host.AssertValue(schema); Host.AssertValue(schemaDef); Host.AssertValue(peeks); Host.AssertValue(predicate); Host.Assert(schema.ColumnCount == schemaDef.Columns.Length); Host.Assert(schema.ColumnCount == peeks.Length); _colCount = schema.ColumnCount; Schema = schema; _getters = new Delegate[_colCount]; for (int c = 0; c < _colCount; c++) { _getters[c] = predicate(c) ? CreateGetter(schema.GetColumnType(c), schemaDef.Columns[c], peeks[c]) : null; } }
/// <summary> /// Create a a map transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are /// not null. /// </summary> /// <param name="env">The host environment</param> /// <param name="source">The dataview upon which we construct the transform</param> /// <param name="mapAction">The action by which we map source to destination columns</param> /// <param name="saveAction">An action that allows us to save state to the serialization stream. May be /// null simultaneously with <paramref name="loadFunc"/>.</param> /// <param name="loadFunc">A function that given the serialization stream and a data view, returns /// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a /// <see cref="MapTransform{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be /// a static non-lambda method that this assembly can legally call. May be null simultaneously with /// <paramref name="saveAction"/>.</param> /// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param> /// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param> public MapTransform(IHostEnvironment env, IDataView source, Action <TSrc, TDst> mapAction, Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) : base(env, RegistrationName, saveAction, loadFunc) { Host.AssertValue(source, "source"); Host.AssertValue(mapAction, "mapAction"); Host.AssertValueOrNull(inputSchemaDefinition); Host.AssertValueOrNull(outputSchemaDefinition); _source = source; _mapAction = mapAction; _inputSchemaDefinition = inputSchemaDefinition; _typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition); var outSchema = InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition); _schema = MergedSchema.Create(_source.Schema, outSchema); }
/// <summary> /// Create a a map transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are /// not null. /// </summary> /// <param name="env">The host environment</param> /// <param name="source">The dataview upon which we construct the transform</param> /// <param name="mapAction">The action by which we map source to destination columns</param> /// <param name="saveAction">An action that allows us to save state to the serialization stream. May be /// null simultaneously with <paramref name="loadFunc"/>.</param> /// <param name="loadFunc">A function that given the serialization stream and a data view, returns /// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a /// <see cref="MapTransform{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be /// a static non-lambda method that this assembly can legally call. May be null simultaneously with /// <paramref name="saveAction"/>.</param> /// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param> /// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param> public MapTransform(IHostEnvironment env, IDataView source, Action <TSrc, TDst> mapAction, Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) : base(env, RegistrationName, saveAction, loadFunc) { Host.AssertValue(source); Host.AssertValue(mapAction); Host.AssertValueOrNull(inputSchemaDefinition); Host.AssertValueOrNull(outputSchemaDefinition); Source = source; _mapAction = mapAction; _inputSchemaDefinition = inputSchemaDefinition; _typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition); var outSchema = outputSchemaDefinition == null ? InternalSchemaDefinition.Create(typeof(TDst), SchemaDefinition.Direction.Write) : InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition); _addedSchema = outSchema; _bindings = new ColumnBindings(Data.Schema.Create(Source.Schema), DataViewConstructionUtils.GetSchemaColumns(outSchema)); }
/// <summary> /// Create a filter transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are /// not null. /// </summary> /// <param name="env">The host environment</param> /// <param name="source">The dataview upon which we construct the transform</param> /// <param name="filterFunc">The function by which we transform source to destination columns and decide whether /// to keep the row.</param> /// <param name="initStateAction">The function that is called once per cursor to initialize state. Can be null.</param> /// <param name="saveAction">An action that allows us to save state to the serialization stream. May be /// null simultaneously with <paramref name="loadFunc"/>.</param> /// <param name="loadFunc">A function that given the serialization stream and a data view, returns /// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a /// <see cref="CustomMappingTransformer{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be /// a static non-lambda method that this assembly can legally call. May be null simultaneously with /// <paramref name="saveAction"/>.</param> /// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param> /// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param> public StatefulFilterTransform(IHostEnvironment env, IDataView source, Func <TSrc, TDst, TState, bool> filterFunc, Action <TState> initStateAction, Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) : base(env, RegistrationName, saveAction, loadFunc) { Host.AssertValue(source, "source"); Host.AssertValue(filterFunc, "filterFunc"); Host.AssertValueOrNull(initStateAction); Host.AssertValueOrNull(inputSchemaDefinition); Host.AssertValueOrNull(outputSchemaDefinition); _source = source; _filterFunc = filterFunc; _initStateAction = initStateAction; _inputSchemaDefinition = inputSchemaDefinition; _typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition); var outSchema = InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition); _addedSchema = outSchema; _bindings = new ColumnBindings(Schema.Create(Source.Schema), DataViewConstructionUtils.GetSchemaColumns(outSchema)); }
public InputRow(IHostEnvironment env, InternalSchemaDefinition schemaDef) : base(env, new SchemaProxy(schemaDef), schemaDef, MakePeeks(schemaDef), c => true) { _position = -1; }
private TypedCursorable(IHostEnvironment env, IDataView data, bool ignoreMissingColumns, InternalSchemaDefinition schemaDefn) { Contracts.AssertValue(env, "env"); _host = env.Register("TypedCursorable"); _host.AssertValue(data); _host.AssertValue(schemaDefn); _data = data; // Get column indices. Throw if there are missing columns (optionally, ignore them). var acceptedCols = new List <InternalSchemaDefinition.Column>(); var indices = new List <int>(); foreach (var col in schemaDefn.Columns) { int colIndex; if (!_data.Schema.TryGetColumnIndex(col.ColumnName, out colIndex)) { if (ignoreMissingColumns) { continue; } throw _host.Except("Column '{0}' not found in the data view", col.ColumnName); } var realColType = _data.Schema.GetColumnType(colIndex); if (!IsCompatibleType(realColType, col.MemberInfo)) { throw _host.Except( "Can't bind the IDataView column '{0}' of type '{1}' to field or property '{2}' of type '{3}'.", col.ColumnName, realColType, col.MemberInfo.Name, col.FieldOrPropertyType.FullName); } acceptedCols.Add(col); indices.Add(colIndex); } _columns = acceptedCols.ToArray(); _columnIndices = indices.ToArray(); _host.Assert(_columns.Length == _columnIndices.Length); int n = _columns.Length; _pokes = new Delegate[n]; _peeks = new Delegate[n]; var schema = _data.Schema; for (int i = 0; i < n; i++) { if (_columns[i].ColumnType.IsVector) { _peeks[i] = ApiUtils.GeneratePeek <TypedCursorable <TRow>, TRow>(_columns[i]); } _pokes[i] = ApiUtils.GeneratePoke <TypedCursorable <TRow>, TRow>(_columns[i]); } }
public InputRow(IHostEnvironment env, InternalSchemaDefinition schemaDef) : base(env, SchemaBuilder.MakeSchema(GetSchemaColumns(schemaDef)), schemaDef, MakePeeks(schemaDef), c => true) { _position = -1; }
public SchemaProxy(InternalSchemaDefinition schemaDefn) { SchemaDefn = schemaDefn; }
public SingleRowLoopDataView(IHostEnvironment env, InternalSchemaDefinition schemaDefn) : base(env, "SingleRowLoopDataView", schemaDefn) { }
public StreamingDataView(IHostEnvironment env, IEnumerable <TRow> data, InternalSchemaDefinition schemaDefn) : base(env, "StreamingDataView", schemaDefn) { Contracts.CheckValue(data, nameof(data)); _data = data; }
public ListDataView(IHostEnvironment env, IList <TRow> data, InternalSchemaDefinition schemaDefn) : base(env, "ListDataView", schemaDefn) { Host.CheckValue(data, nameof(data)); _data = data; }