private MergedSchema(ISchema inputSchema, InternalSchemaDefinition addedSchema)
: base(inputSchema)
{
Contracts.AssertValue(inputSchema);
Contracts.AssertValue(addedSchema);
Contracts.Assert(addedSchema.Columns.Length == 0);
AddedSchema = addedSchema;
}
/// <summary>
/// Create a Cursorable object on a given data view.
/// </summary>
/// <param name="env">Host enviroment.</param>
/// <param name="data">The underlying data view.</param>
/// <param name="ignoreMissingColumns">Whether to ignore missing columns in the data view.</param>
/// <param name="schemaDefinition">The optional user-provided schema.</param>
/// <returns>The constructed Cursorable.</returns>
public static TypedCursorable <TRow> Create(IHostEnvironment env, IDataView data, bool ignoreMissingColumns, SchemaDefinition schemaDefinition)
{
Contracts.AssertValue(env);
env.AssertValue(data);
env.AssertValueOrNull(schemaDefinition);
var intSchemaDefn = InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition);
return(new TypedCursorable <TRow>(env, data, ignoreMissingColumns, intSchemaDefn));
}
private MergedSchema(ISchema inputSchema, InternalSchemaDefinition addedSchema, string[] newColumnNames)
: base(inputSchema, true, newColumnNames)
{
Contracts.AssertValue(inputSchema);
Contracts.AssertValue(addedSchema);
Contracts.AssertNonEmpty(newColumnNames);
Contracts.Assert(addedSchema.Columns.Length == newColumnNames.Length);
AddedSchema = addedSchema;
}
public static InputRow <TRow> CreateInputRow <TRow>(IHostEnvironment env, SchemaDefinition schemaDefinition = null)
where TRow : class
{
Contracts.AssertValue(env);
env.AssertValueOrNull(schemaDefinition);
var internalSchemaDefn = schemaDefinition == null
? InternalSchemaDefinition.Create(typeof(TRow), SchemaDefinition.Direction.Read)
: InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition);
return(new InputRow <TRow>(env, internalSchemaDefn));
}
/// <summary>
/// The 'reapply' constructor.
/// </summary>
private StatefulFilterTransform(IHostEnvironment env, StatefulFilterTransform <TSrc, TDst, TState> transform, IDataView newSource)
: base(env, RegistrationName, transform)
{
Host.AssertValue(transform);
Host.AssertValue(newSource);
_source = newSource;
_filterFunc = transform._filterFunc;
_typedSource = TypedCursorable <TSrc> .Create(Host, newSource, false, transform._inputSchemaDefinition);
_addedSchema = transform._addedSchema;
_bindings = new ColumnBindings(Schema.Create(newSource.Schema), DataViewConstructionUtils.GetSchemaColumns(_addedSchema));
}
public static MergedSchema Create(ISchema inputSchema, InternalSchemaDefinition addedSchema)
{
Contracts.AssertValue(inputSchema);
Contracts.AssertValue(addedSchema);
if (addedSchema.Columns.Length > 0)
{
var newColumnNames = addedSchema.Columns.Select(x => x.ColumnName).ToArray();
return(new MergedSchema(inputSchema, addedSchema, newColumnNames));
}
return(new MergedSchema(inputSchema, addedSchema));
}
/// <summary>
/// Returns whether the column type <paramref name="colType"/> can be bound to field <paramref name="memberInfo"/>.
/// They must both be vectors or scalars, and the raw data kind should match.
/// </summary>
private static bool IsCompatibleType(ColumnType colType, MemberInfo memberInfo)
{
InternalSchemaDefinition.GetVectorAndKind(memberInfo, out bool isVector, out DataKind kind);
if (isVector)
{
return(colType.IsVector && colType.ItemType.RawKind == kind);
}
else
{
return(!colType.IsVector && colType.RawKind == kind);
}
}
public static StreamingDataView <TRow> CreateFromEnumerable <TRow>(IHostEnvironment env, IEnumerable <TRow> data,
SchemaDefinition schemaDefinition = null)
where TRow : class
{
Contracts.AssertValue(env);
env.AssertValue(data);
env.AssertValueOrNull(schemaDefinition);
var internalSchemaDefn = schemaDefinition == null
? InternalSchemaDefinition.Create(typeof(TRow), SchemaDefinition.Direction.Read)
: InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition);
return(new StreamingDataView <TRow>(env, data, internalSchemaDefn));
}
/// <summary>
/// The 'reapply' constructor.
/// </summary>
private MapTransform(IHostEnvironment env, MapTransform <TSrc, TDst> transform, IDataView newSource)
: base(env, RegistrationName, transform)
{
Host.AssertValue(transform);
Host.AssertValue(newSource);
Source = newSource;
_mapAction = transform._mapAction;
_typedSource = TypedCursorable <TSrc> .Create(Host, newSource, false, transform._inputSchemaDefinition);
_addedSchema = transform._addedSchema;
_bindings = new ColumnBindings(Data.Schema.Create(newSource.Schema), DataViewConstructionUtils.GetSchemaColumns(_addedSchema));
}
public static IDataView CreateFromList <TRow>(IHostEnvironment env, IList <TRow> data,
SchemaDefinition schemaDefinition = null)
where TRow : class
{
Contracts.AssertValue(env);
env.AssertValue(data);
env.AssertValueOrNull(schemaDefinition);
var internalSchemaDefn = schemaDefinition == null
? InternalSchemaDefinition.Create(typeof(TRow))
: InternalSchemaDefinition.Create(typeof(TRow), schemaDefinition);
return(new ListDataView <TRow>(env, data, internalSchemaDefn));
}
private static Delegate[] MakePeeks(InternalSchemaDefinition schemaDef)
{
var peeks = new Delegate[schemaDef.Columns.Length];
for (var i = 0; i < peeks.Length; i++)
{
var currentColumn = schemaDef.Columns[i];
peeks[i] = currentColumn.IsComputed
? currentColumn.Generator
: ApiUtils.GeneratePeek <InputRow <TRow>, TRow>(currentColumn);
}
return(peeks);
}
/// <summary>
/// Returns whether the column type <paramref name="colType"/> can be bound to field <paramref name="fieldInfo"/>.
/// They must both be vectors or scalars, and the raw data kind should match.
/// </summary>
private static bool IsCompatibleType(ColumnType colType, FieldInfo fieldInfo)
{
bool isVector;
DataKind kind;
InternalSchemaDefinition.GetVectorAndKind(fieldInfo, out isVector, out kind);
if (isVector)
{
return(colType.IsVector && colType.ItemType.RawKind == kind);
}
else
{
return(!colType.IsVector && colType.RawKind == kind);
}
}
protected DataViewBase(IHostEnvironment env, string name, InternalSchemaDefinition schemaDefn)
{
Contracts.AssertValue(env);
env.AssertNonWhiteSpace(name);
Host = env.Register(name);
Host.AssertValue(schemaDefn);
_schema = new SchemaProxy(schemaDefn);
int n = _schema.SchemaDefn.Columns.Length;
_peeks = new Delegate[n];
for (var i = 0; i < n; i++)
{
var currentColumn = _schema.SchemaDefn.Columns[i];
_peeks[i] = currentColumn.IsComputed
? currentColumn.Generator
: ApiUtils.GeneratePeek <DataViewBase <TRow>, TRow>(currentColumn);
}
}
internal static Schema.DetachedColumn[] GetSchemaColumns(InternalSchemaDefinition schemaDefn)
{
Contracts.AssertValue(schemaDefn);
var columns = new Schema.DetachedColumn[schemaDefn.Columns.Length];
for (int i = 0; i < columns.Length; i++)
{
var col = schemaDefn.Columns[i];
var meta = new MetadataBuilder();
foreach (var kvp in col.Metadata)
{
meta.Add(kvp.Value.Kind, kvp.Value.MetadataType, kvp.Value.GetGetterDelegate());
}
columns[i] = new Schema.DetachedColumn(col.ColumnName, col.ColumnType, meta.GetMetadata());
}
return(columns);
}
public InputRowBase(IHostEnvironment env, Schema schema, InternalSchemaDefinition schemaDef, Delegate[] peeks, Func <int, bool> predicate)
{
Contracts.AssertValue(env);
Host = env.Register("Row");
Host.AssertValue(schema);
Host.AssertValue(schemaDef);
Host.AssertValue(peeks);
Host.AssertValue(predicate);
Host.Assert(schema.ColumnCount == schemaDef.Columns.Length);
Host.Assert(schema.ColumnCount == peeks.Length);
_colCount = schema.ColumnCount;
Schema = schema;
_getters = new Delegate[_colCount];
for (int c = 0; c < _colCount; c++)
{
_getters[c] = predicate(c) ? CreateGetter(schema.GetColumnType(c), schemaDef.Columns[c], peeks[c]) : null;
}
}
/// <summary>
/// Create a a map transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are
/// not null.
/// </summary>
/// <param name="env">The host environment</param>
/// <param name="source">The dataview upon which we construct the transform</param>
/// <param name="mapAction">The action by which we map source to destination columns</param>
/// <param name="saveAction">An action that allows us to save state to the serialization stream. May be
/// null simultaneously with <paramref name="loadFunc"/>.</param>
/// <param name="loadFunc">A function that given the serialization stream and a data view, returns
/// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a
/// <see cref="MapTransform{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be
/// a static non-lambda method that this assembly can legally call. May be null simultaneously with
/// <paramref name="saveAction"/>.</param>
/// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param>
/// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param>
public MapTransform(IHostEnvironment env, IDataView source, Action <TSrc, TDst> mapAction,
Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc,
SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
: base(env, RegistrationName, saveAction, loadFunc)
{
Host.AssertValue(source, "source");
Host.AssertValue(mapAction, "mapAction");
Host.AssertValueOrNull(inputSchemaDefinition);
Host.AssertValueOrNull(outputSchemaDefinition);
_source = source;
_mapAction = mapAction;
_inputSchemaDefinition = inputSchemaDefinition;
_typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition);
var outSchema = InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition);
_schema = MergedSchema.Create(_source.Schema, outSchema);
}
/// <summary>
/// Create a a map transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are
/// not null.
/// </summary>
/// <param name="env">The host environment</param>
/// <param name="source">The dataview upon which we construct the transform</param>
/// <param name="mapAction">The action by which we map source to destination columns</param>
/// <param name="saveAction">An action that allows us to save state to the serialization stream. May be
/// null simultaneously with <paramref name="loadFunc"/>.</param>
/// <param name="loadFunc">A function that given the serialization stream and a data view, returns
/// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a
/// <see cref="MapTransform{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be
/// a static non-lambda method that this assembly can legally call. May be null simultaneously with
/// <paramref name="saveAction"/>.</param>
/// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param>
/// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param>
public MapTransform(IHostEnvironment env, IDataView source, Action <TSrc, TDst> mapAction,
Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc,
SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
: base(env, RegistrationName, saveAction, loadFunc)
{
Host.AssertValue(source);
Host.AssertValue(mapAction);
Host.AssertValueOrNull(inputSchemaDefinition);
Host.AssertValueOrNull(outputSchemaDefinition);
Source = source;
_mapAction = mapAction;
_inputSchemaDefinition = inputSchemaDefinition;
_typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition);
var outSchema = outputSchemaDefinition == null
? InternalSchemaDefinition.Create(typeof(TDst), SchemaDefinition.Direction.Write)
: InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition);
_addedSchema = outSchema;
_bindings = new ColumnBindings(Data.Schema.Create(Source.Schema), DataViewConstructionUtils.GetSchemaColumns(outSchema));
}
/// <summary>
/// Create a filter transform that is savable iff <paramref name="saveAction"/> and <paramref name="loadFunc"/> are
/// not null.
/// </summary>
/// <param name="env">The host environment</param>
/// <param name="source">The dataview upon which we construct the transform</param>
/// <param name="filterFunc">The function by which we transform source to destination columns and decide whether
/// to keep the row.</param>
/// <param name="initStateAction">The function that is called once per cursor to initialize state. Can be null.</param>
/// <param name="saveAction">An action that allows us to save state to the serialization stream. May be
/// null simultaneously with <paramref name="loadFunc"/>.</param>
/// <param name="loadFunc">A function that given the serialization stream and a data view, returns
/// an <see cref="ITransformTemplate"/>. The intent is, this returned object should itself be a
/// <see cref="CustomMappingTransformer{TSrc,TDst}"/>, but this is not strictly necessary. This delegate should be
/// a static non-lambda method that this assembly can legally call. May be null simultaneously with
/// <paramref name="saveAction"/>.</param>
/// <param name="inputSchemaDefinition">The schema definition overrides for <typeparamref name="TSrc"/></param>
/// <param name="outputSchemaDefinition">The schema definition overrides for <typeparamref name="TDst"/></param>
public StatefulFilterTransform(IHostEnvironment env, IDataView source, Func <TSrc, TDst, TState, bool> filterFunc,
Action <TState> initStateAction,
Action <BinaryWriter> saveAction, LambdaTransform.LoadDelegate loadFunc,
SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
: base(env, RegistrationName, saveAction, loadFunc)
{
Host.AssertValue(source, "source");
Host.AssertValue(filterFunc, "filterFunc");
Host.AssertValueOrNull(initStateAction);
Host.AssertValueOrNull(inputSchemaDefinition);
Host.AssertValueOrNull(outputSchemaDefinition);
_source = source;
_filterFunc = filterFunc;
_initStateAction = initStateAction;
_inputSchemaDefinition = inputSchemaDefinition;
_typedSource = TypedCursorable <TSrc> .Create(Host, Source, false, inputSchemaDefinition);
var outSchema = InternalSchemaDefinition.Create(typeof(TDst), outputSchemaDefinition);
_addedSchema = outSchema;
_bindings = new ColumnBindings(Schema.Create(Source.Schema), DataViewConstructionUtils.GetSchemaColumns(outSchema));
}
public InputRow(IHostEnvironment env, InternalSchemaDefinition schemaDef)
: base(env, new SchemaProxy(schemaDef), schemaDef, MakePeeks(schemaDef), c => true)
{
_position = -1;
}
private TypedCursorable(IHostEnvironment env, IDataView data, bool ignoreMissingColumns, InternalSchemaDefinition schemaDefn)
{
Contracts.AssertValue(env, "env");
_host = env.Register("TypedCursorable");
_host.AssertValue(data);
_host.AssertValue(schemaDefn);
_data = data;
// Get column indices. Throw if there are missing columns (optionally, ignore them).
var acceptedCols = new List <InternalSchemaDefinition.Column>();
var indices = new List <int>();
foreach (var col in schemaDefn.Columns)
{
int colIndex;
if (!_data.Schema.TryGetColumnIndex(col.ColumnName, out colIndex))
{
if (ignoreMissingColumns)
{
continue;
}
throw _host.Except("Column '{0}' not found in the data view", col.ColumnName);
}
var realColType = _data.Schema.GetColumnType(colIndex);
if (!IsCompatibleType(realColType, col.MemberInfo))
{
throw _host.Except(
"Can't bind the IDataView column '{0}' of type '{1}' to field or property '{2}' of type '{3}'.",
col.ColumnName, realColType, col.MemberInfo.Name, col.FieldOrPropertyType.FullName);
}
acceptedCols.Add(col);
indices.Add(colIndex);
}
_columns = acceptedCols.ToArray();
_columnIndices = indices.ToArray();
_host.Assert(_columns.Length == _columnIndices.Length);
int n = _columns.Length;
_pokes = new Delegate[n];
_peeks = new Delegate[n];
var schema = _data.Schema;
for (int i = 0; i < n; i++)
{
if (_columns[i].ColumnType.IsVector)
{
_peeks[i] = ApiUtils.GeneratePeek <TypedCursorable <TRow>, TRow>(_columns[i]);
}
_pokes[i] = ApiUtils.GeneratePoke <TypedCursorable <TRow>, TRow>(_columns[i]);
}
}
public InputRow(IHostEnvironment env, InternalSchemaDefinition schemaDef)
: base(env, SchemaBuilder.MakeSchema(GetSchemaColumns(schemaDef)), schemaDef, MakePeeks(schemaDef), c => true)
{
_position = -1;
}
public SchemaProxy(InternalSchemaDefinition schemaDefn)
{
SchemaDefn = schemaDefn;
}
public SingleRowLoopDataView(IHostEnvironment env, InternalSchemaDefinition schemaDefn)
: base(env, "SingleRowLoopDataView", schemaDefn)
{
}
public StreamingDataView(IHostEnvironment env, IEnumerable <TRow> data, InternalSchemaDefinition schemaDefn)
: base(env, "StreamingDataView", schemaDefn)
{
Contracts.CheckValue(data, nameof(data));
_data = data;
}
public ListDataView(IHostEnvironment env, IList <TRow> data, InternalSchemaDefinition schemaDefn)
: base(env, "ListDataView", schemaDefn)
{
Host.CheckValue(data, nameof(data));
_data = data;
}
