public override ValueGetter <TValue> GetGetter <TValue>(int col)
{
Ch.Check(IsColumnActive(col));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, col);
if (isSrc)
{
return(Input.GetGetter <TValue>(index));
}
Ch.AssertValue(_getters);
var getter = _getters[index];
Ch.AssertValue(getter);
if (getter is ValueGetter <TValue> fn)
{
return(fn);
}
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
public RandCursor(AppendRowsDataView parent, IEnumerable <DataViewSchema.Column> columnsNeeded, Random rand, int[] counts)
: base(parent)
{
Ch.AssertValue(rand);
_rand = rand;
Ch.AssertValue(counts);
Ch.Assert(Sources.Length == counts.Length);
_cursorSet = new DataViewRowCursor[counts.Length];
for (int i = 0; i < counts.Length; i++)
{
Ch.Assert(counts[i] >= 0);
_cursorSet[i] = parent._sources[i].GetRowCursor(columnsNeeded, RandomUtils.Create(_rand));
}
_sampler = new MultinomialWithoutReplacementSampler(Ch, counts, rand);
_currentSourceIndex = -1;
foreach (var col in columnsNeeded)
{
Getters[col.Index] = CreateGetter(col.Index);
}
}
public Cursor(ParquetLoader parent, Func <int, bool> predicate, IRandom rand)
: base(parent._host)
{
Ch.AssertValue(predicate);
_loader = parent;
_fileStream = parent._parquetStream;
_parquetConversions = new ParquetConversions(Ch);
_rand = rand;
// Create Getter delegates
Utils.BuildSubsetMaps(Schema.ColumnCount, predicate, out _actives, out _colToActivesIndex);
_readerOptions = new ReaderOptions
{
Count = _loader._columnChunkReadSize,
Columns = _loader._columnsLoaded.Select(i => i.Name).ToArray()
};
// The number of blocks is calculated based on the specified rows in a block (defaults to 1M).
// Since we want to shuffle the blocks in addition to shuffling the rows in each block, checks
// are put in place to ensure we can produce a shuffle order for the blocks.
var numBlocks = MathUtils.DivisionCeiling((long)parent.GetRowCount(), _readerOptions.Count);
if (numBlocks > int.MaxValue)
{
throw _loader._host.ExceptParam(nameof(Arguments.ColumnChunkReadSize), "Error due to too many blocks. Try increasing block size.");
}
var blockOrder = CreateOrderSequence((int)numBlocks);
_blockEnumerator = blockOrder.GetEnumerator();
_dataSetEnumerator = Enumerable.Empty <int>().GetEnumerator();
_columnValues = new IList[_actives.Length];
_getters = new Delegate[_actives.Length];
for (int i = 0; i < _actives.Length; ++i)
{
int columnIndex = _actives[i];
_getters[i] = CreateGetterDelegate(columnIndex);
}
}
public ValueGetter <TValue> GetGetter <TValue>(int col)
{
Ch.CheckParam(0 <= col && col < _bindings.ColumnCount, nameof(col));
Ch.CheckParam(IsColumnActive(col), nameof(col));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, col);
if (isSrc)
{
return(Input.GetGetter <TValue>(index));
}
Ch.Assert(index == 0);
Delegate idGetter = Input.GetIdGetter();
Ch.AssertValue(idGetter);
var fn = idGetter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.CheckParam(column.Index < _bindings.ColumnCount, nameof(column));
Ch.CheckParam(IsColumnActive(column), nameof(column.Index));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(Input.Schema[index]));
}
Ch.Assert(index == 0);
Delegate idGetter = Input.GetIdGetter();
Ch.AssertValue(idGetter);
var fn = idGetter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(column));
}
Ch.AssertValue(_getters);
var getter = _getters[index];
Ch.AssertValue(getter);
if (getter is ValueGetter <TValue> fn)
{
return(fn);
}
throw Ch.Except($"Invalid TValue in GetGetter: '{typeof(TValue)}', " +
$"expected type: '{getter.GetType().GetGenericArguments().First()}'.");
}
public Cursor(IChannelProvider provider, RowToRowScorerBase parent, RowCursor input, bool[] active, Func <int, bool> predicateMapper)
: base(provider, input)
{
Ch.AssertValue(parent);
Ch.AssertValue(active);
Ch.AssertValue(predicateMapper);
_bindings = parent.GetBindings();
Schema = parent.OutputSchema;
Ch.Assert(active.Length == _bindings.ColumnCount);
_active = active;
_output = _bindings.RowMapper.GetRow(input, predicateMapper);
try
{
Ch.Assert(_output.Schema == _bindings.RowMapper.OutputSchema);
_getters = parent.GetGetters(_output, iinfo => active[_bindings.MapIinfoToCol(iinfo)]);
}
catch (Exception)
{
_output.Dispose();
throw;
}
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(Input.Schema[index]));
}
Ch.AssertValue(_getters);
var getter = _getters[index];
Ch.Assert(getter != null);
var fn = getter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
public Cursor(IChannelProvider provider, IDataTransform view, Bindings bindings, Func <int, bool> predicate, Random rand)
: base(provider)
{
Ch.AssertValue(view);
Ch.AssertValueOrNull(rand);
Ch.Assert(view.Schema.Count >= 0);
_view = view;
_bindings = bindings;
_cursor = view.Source.GetRowCursorForAllColumns();
_active = new BitArray(view.Schema.Count);
if (predicate == null)
{
_active.SetAll(true);
}
else
{
for (int i = 0; i < view.Schema.Count; ++i)
{
_active[i] = predicate(i);
}
}
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
if (isSrc)
{
Contracts.AssertValue(_input);
return(_input.GetGetter <TValue>(_input.Schema[col]));
}
Ch.AssertValue(_getters);
var getter = _getters[col];
Ch.Assert(getter != null);
var fn = getter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
// Note that we don't filter out rows with parsing issues since it's not acceptable to
// produce a different set of rows when subsetting columns. Any parsing errors need to be
// translated to NaN, not result in skipping the row. We should produce some diagnostics
// to alert the user to the issues.
private Cursor(TextLoader parent, ParseStats stats, bool[] active, LineReader reader, int srcNeeded, int cthd)
: base(parent._host)
{
Ch.Assert(active == null || active.Length == parent._bindings.Infos.Length);
Ch.AssertValue(reader);
Ch.AssertValue(stats);
Ch.Assert(srcNeeded >= 0);
Ch.Assert(cthd > 0);
_total = -1;
_batch = -1;
_bindings = parent._bindings;
_parser = parent._parser;
_active = active;
_reader = reader;
_stats = stats;
_srcNeeded = srcNeeded;
ParallelState state = null;
if (cthd > 1)
{
state = new ParallelState(this, out _rows, cthd);
}
else
{
_rows = _parser.CreateRowSet(_stats, 1, _active);
}
try
{
_getters = new Delegate[_bindings.Infos.Length];
for (int i = 0; i < _getters.Length; i++)
{
if (_active != null && !_active[i])
{
continue;
}
ColumnPipe v = _rows.Pipes[i];
Ch.Assert(v != null);
_getters[i] = v.GetGetter();
Ch.Assert(_getters[i] != null);
}
if (state != null)
{
_ator = ParseParallel(state).GetEnumerator();
state = null;
}
else
{
_ator = ParseSequential().GetEnumerator();
}
}
finally
{
if (state != null)
{
state.Dispose();
}
}
}
