public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, IRandom rand = null)
{
consolidator = null;
var cursor = new Cursor(_host, this, _files, needCol, rand);
return(new IRowCursor[] { cursor });
}
public override RowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, Random rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var bindings = GetBindings();
Func <int, bool> predicateInput;
Func <int, bool> predicateMapper;
var active = GetActive(bindings, predicate, out predicateInput, out predicateMapper);
var inputs = Source.GetRowCursorSet(out consolidator, predicateInput, n, rand);
Contracts.AssertNonEmpty(inputs);
if (inputs.Length == 1 && n > 1 && WantParallelCursors(predicate) && (Source.GetRowCount() ?? int.MaxValue) > n)
{
inputs = DataViewUtils.CreateSplitCursors(out consolidator, Host, inputs[0], n);
}
Contracts.AssertNonEmpty(inputs);
var cursors = new RowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new Cursor(Host, this, inputs[i], active, predicateMapper);
}
return(cursors);
}
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, IRandom rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var inputPred = _bindings.GetDependencies(predicate);
var active = _bindings.GetActive(predicate);
IRowCursor input;
if (n > 1 && ShouldUseParallelCursors(predicate) != false)
{
var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n);
Host.AssertNonEmpty(inputs);
if (inputs.Length != 1)
{
var cursors = new IRowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new RowCursor(Host, _bindings, inputs[i], active);
}
return(cursors);
}
input = inputs[0];
}
else
{
input = Source.GetRowCursor(inputPred);
}
consolidator = null;
return(new IRowCursor[] { new RowCursor(Host, _bindings, input, active) });
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, IRandom rand = null)
{
_host.CheckValue(predicate, nameof(predicate));
_host.CheckValueOrNull(rand);
return(View.GetRowCursorSet(out consolidator, predicate, n, rand));
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
consolidator = null;
return(new IRowCursor[] { GetRowCursor(predicate, rand) });
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, Random rand = null)
{
_host.CheckValue(needCol, nameof(needCol));
_host.CheckValueOrNull(rand);
consolidator = null;
return(new[] { new Cursor(_host, Schema, needCol) });
}
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
var cursor = GetRowCursorCore(predicate, rand);
consolidator = null;
return(new IRowCursor[] { cursor });
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
ComputeNearestNeighbors();
_host.AssertValue(_input, "_input");
var schema = _input.Schema;
if (predicate(_input.Schema.ColumnCount))
{
int featureIndex;
if (!schema.TryGetColumnIndex(_args.column, out featureIndex))
{
throw _host.Except("Unable to find column '{0}'.", _args.column);
}
var res = _input.GetRowCursorSet(out consolidator, predicate, n, rand)
.Select(c => new NearestNeighborsCursor(c, this, i => PredicatePropagation(i, featureIndex, predicate), featureIndex)).ToArray();
consolidator = new Consolidator();
return(res);
}
else
{
// The new column is not required. We do not need to compute it. But we need to keep the same schema.
return(_input.GetRowCursorSet(out consolidator, predicate, n, rand)
.Select(c => new SameCursor(c, Schema))
.ToArray());
}
}
public sealed override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, IRandom rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var inputPred = _bindings.GetDependencies(predicate);
var active = _bindings.GetActive(predicate);
var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n, rand);
Host.AssertNonEmpty(inputs);
if (inputs.Length == 1 && n > 1 && _bindings.AnyNewColumnsActive(predicate))
{
inputs = DataViewUtils.CreateSplitCursors(out consolidator, Host, inputs[0], n);
}
Host.AssertNonEmpty(inputs);
var cursors = new IRowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new RowCursor(this, inputs[i], active);
}
return(cursors);
}
public RowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, Random rand = null)
{
_host.CheckValue(predicate, nameof(predicate));
_host.CheckValueOrNull(rand);
consolidator = null;
return(new RowCursor[] { new Cursor(_host, this, predicate, rand) });
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
TrainTransform();
_host.AssertValue(_reversedMapping, "_reversedMapping");
var cursors = _input.GetRowCursorSet(out consolidator, predicate, n, rand);
return(cursors.Select(c => new DBScanCursor(this, c)).ToArray());
}
public override RowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func<int, bool> predicate,
int n, Random rand = null)
{
var activeInput = _schemaImpl.GetActiveInput(predicate);
var inputCursors = Source.GetRowCursorSet(out consolidator, col => activeInput[col], n, null);
return Utils.BuildArray<RowCursor>(inputCursors.Length,
x => new Cursor(Host, inputCursors[x], _schemaImpl, predicate));
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
TrainTransform();
_host.AssertValue(_Results, "_Results");
var cursors = _input.GetRowCursorSet(out consolidator, predicate, n, rand);
return(cursors.Select(c => new OpticsOrderingCursor(this, c)).ToArray());
}
public static IRowCursor[] CreateSet(out IRowCursorConsolidator consolidator,
TextLoader parent, IMultiStreamSource files, bool[] active, int n)
{
// Note that files is allowed to be empty.
Contracts.AssertValue(parent);
Contracts.AssertValue(files);
Contracts.Assert(active == null || active.Length == parent._bindings.Infos.Length);
int srcNeeded;
int cthd;
SetupCursor(parent, active, n, out srcNeeded, out cthd);
Contracts.Assert(cthd > 0);
var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, cthd);
var stats = new ParseStats(parent._host, cthd);
if (cthd <= 1)
{
consolidator = null;
return(new IRowCursor[1] {
new Cursor(parent, stats, active, reader, srcNeeded, 1)
});
}
consolidator = new Consolidator(cthd);
var cursors = new IRowCursor[cthd];
try
{
for (int i = 0; i < cursors.Length; i++)
{
cursors[i] = new Cursor(parent, stats, active, reader, srcNeeded, 1);
}
var result = cursors;
cursors = null;
return(result);
}
finally
{
if (cursors != null)
{
foreach (var curs in cursors)
{
if (curs != null)
{
curs.Dispose();
}
else
{
reader.Release();
stats.Release();
}
}
}
}
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
ComputeStatistics();
_host.AssertValue(_input, "_input");
var cursors = _input.GetRowCursorSet(out consolidator, i => PredicatePropagation(i, predicate), n, rand);
var res = cursors.Select(c => new ScalerCursor(c, this, i => PredicatePropagation(i, predicate))).ToArray();
consolidator = new Consolidator();
return(res);
}
public RowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, Random rand = null)
{
_host.CheckValue(predicate, nameof(predicate));
if (HasRowData)
{
return(_schemaEntry.GetView().GetRowCursorSet(out consolidator, predicate, n, rand));
}
consolidator = null;
return(new RowCursor[] { GetRowCursor(predicate, rand) });
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, Random rand = null)
{
Contracts.CheckValue(predicate, nameof(predicate));
Contracts.CheckParam(n >= 0, nameof(n));
Contracts.CheckValueOrNull(rand);
// This transform is stateful, its contract is to allocate exactly one state object per cursor and call the filter function
// on every row in sequence. Therefore, parallel cursoring is not possible.
consolidator = null;
return(new[] { GetRowCursor(predicate, rand) });
}
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, IRandom rand = null)
{
if (_transform == null)
{
lock (_lock)
if (_transform == null)
{
_transform = CreateTemplatedTransform();
}
}
Host.AssertValue(_transform, "_transform");
Host.AssertValue(_trend, "_trend");
return(_transform.GetRowCursorSet(out consolidator, needCol, n, rand));
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
if (predicate(_input.Schema.ColumnCount))
{
var cursors = _input.GetRowCursorSet(out consolidator, i => PredicatePropagation(i, predicate), n, rand);
return(cursors.Select(c => new PolynomialCursor <TInput>(this, c, i => PredicatePropagation(i, predicate), _args, _inputCol, _multiplication)).ToArray());
}
else
{
// The new column is not required. We do not need to compute it. But we need to keep the same schema.
return(_input.GetRowCursorSet(out consolidator, predicate, n, rand)
.Select(c => new SameCursor(c, Schema))
.ToArray());
}
}
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, Random rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var inputPred = _bindings.GetDependencies(predicate);
IRowCursor[] cursors = Source.GetRowCursorSet(out consolidator, inputPred, n, rand);
bool active = predicate(_bindings.MapIinfoToCol(0));
for (int c = 0; c < cursors.Length; ++c)
{
cursors[c] = new RowCursor(Host, _bindings, cursors[c], active);
}
return(cursors);
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, IRandom rand = null)
{
if (predicate(_source.Schema.ColumnCount))
{
var cursors = _source.GetRowCursorSet(out consolidator, i => PredicatePropagation(i, predicate), n, rand);
return(cursors.Select(c => new LambdaCursor(this, c)).ToArray());
}
else
{
// The new column is not required. We do not need to compute it. But we need to keep the same schema.
return(_source.GetRowCursorSet(out consolidator, predicate, n, rand)
.Select(c => new SameCursor(c, Schema))
.ToArray());
}
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var activeInputs = _schema.GetActiveInput(predicate);
Func <int, bool> srcPredicate = c => activeInputs[c];
var inputs = _typedSource.GetCursorSet(out consolidator, srcPredicate, n, rand);
Host.AssertNonEmpty(inputs);
var cursors = new IRowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new Cursor(Host, this, inputs[i], predicate);
}
return(cursors);
}
public sealed override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, IRandom rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
var inputPred = _bindings.GetDependencies(predicate);
var active = _bindings.GetActive(predicate);
var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n, rand);
Host.AssertNonEmpty(inputs);
// No need to split if this is given 1 input cursor.
var cursors = new IRowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new RowCursor(Host, _bindings, inputs[i], active);
}
return(cursors);
}
/// <summary>
/// Create a set of cursors with additional active columns.
/// </summary>
/// <param name="consolidator">The consolidator for the original row cursors</param>
/// <param name="additionalColumnsPredicate">Predicate that denotes which additional columns to include in the cursor,
/// in addition to the columns that are needed for populating the <typeparamref name="TRow"/> object.</param>
/// <param name="n">Number of cursors to create</param>
/// <param name="rand">Random generator to use</param>
public IRowCursor <TRow>[] GetCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> additionalColumnsPredicate, int n, IRandom rand)
{
_host.CheckValue(additionalColumnsPredicate, nameof(additionalColumnsPredicate));
_host.CheckValueOrNull(rand);
Func <int, bool> inputPredicate = col => _columnIndices.Contains(col) || additionalColumnsPredicate(col);
var inputs = _data.GetRowCursorSet(out consolidator, inputPredicate, n, rand);
_host.AssertNonEmpty(inputs);
if (inputs.Length == 1 && n > 1)
{
inputs = DataViewUtils.CreateSplitCursors(out consolidator, _host, inputs[0], n);
}
_host.AssertNonEmpty(inputs);
return(inputs
.Select(rc => (IRowCursor <TRow>)(new TypedCursor(this, rc)))
.ToArray());
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
int index;
if (!Source.Schema.TryGetColumnIndex(_parent.InputName, out index))
{
throw _host.Except("Unable to find column '{0}' in input schema.", _parent.InputName);
}
if (predicate(index))
{
var cursors = Source.GetRowCursorSet(out consolidator, i => PredicatePropagation(i, index, predicate), n, rand);
return(cursors.Select(c => new MemoryCursor <TSrc, TDst>(this, c, index)).ToArray());
}
else
{
// The new column is not required. We do not need to compute it. But we need to keep the same schema.
return(Source.GetRowCursorSet(out consolidator, predicate, n, rand)
.Select(c => new SameCursor(c, Schema))
.ToArray());
}
}
public override RowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator,
Func <int, bool> predicate, int n, Random rand = null)
{
Host.CheckValue(predicate, nameof(predicate));
Host.CheckValueOrNull(rand);
bool[] active;
Func <int, bool> inputPred = GetActive(predicate, out active);
var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n, rand);
Host.AssertNonEmpty(inputs);
// No need to split if this is given 1 input cursor.
var cursors = new RowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = CreateCursorCore(inputs[i], active);
}
return(cursors);
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, IRandom rand = null)
{
var cur = GetRowCursor(needCol, rand);
consolidator = new Consolidator();
if (n >= 2)
{
var setColumns = new HashSet <int>(_columns);
var res = new IRowCursor[n];
var empty = new EmptyCursor(this,
col => setColumns.Contains(col) || needCol(col) || (_otherValues != null && _otherValues.IsColumnActive(col)));
for (int i = 0; i < n; ++i)
{
res[i] = i == 0 ? cur : empty;
}
return(res);
}
else
{
return new IRowCursor[] { cur }
};
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
var host = new ConsoleEnvironment().Register("Estimate n threads");
n = DataViewUtils.GetThreadCount(host, n);
if (n <= 1)
{
consolidator = null;
return(new IRowCursor[] { GetRowCursor(predicate, rand) });
}
else
{
var cursors = Source.GetRowCursorSet(out consolidator, i => predicate(i) || predicate(SchemaHelper.NeedColumn(_columnMapping, i)),
n, rand);
for (int i = 0; i < cursors.Length; ++i)
{
cursors[i] = new AddRandomCursor(this, cursors[i]);
}
return(cursors);
}
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, IRandom rand = null)
{
var cur = GetRowCursor(needCol, rand);
consolidator = new Consolidator();
if (n >= 2)
{
// This trick avoids the cursor to be split into multiple later.
var res = new IRowCursor[n];
var empty = new EmptyCursor(this,
col => col == _column || needCol(col) ||
(_otherValues != null && _otherValues.IsColumnActive(col)));
for (int i = 0; i < n; ++i)
{
res[i] = i == 0 ? cur : empty;
}
return(res);
}
else
{
return new IRowCursor[] { cur }
};
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> needCol, int n, IRandom rand = null)
{
_host.CheckValue(needCol, nameof(needCol));
_host.CheckValueOrNull(rand);
// Build out the active state for the input
var inputPred = GetDependencies(needCol);
var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n, rand);
// Build out the acitve state for the output
var active = Utils.BuildArray(_mapper.Schema.ColumnCount, needCol);
_host.AssertNonEmpty(inputs);
// No need to split if this is given 1 input cursor.
var cursors = new IRowCursor[inputs.Length];
for (int i = 0; i < inputs.Length; i++)
{
cursors[i] = new RowCursor(_host, _mapper, inputs[i], active);
}
return(cursors);
}
