public static IRowCursor[] CreateSet(out IRowCursorConsolidator consolidator,
TextLoader parent, IMultiStreamSource files, bool[] active, int n)
{
// Note that files is allowed to be empty.
Contracts.AssertValue(parent);
Contracts.AssertValue(files);
Contracts.Assert(active == null || active.Length == parent._bindings.Infos.Length);
int srcNeeded;
int cthd;
SetupCursor(parent, active, n, out srcNeeded, out cthd);
Contracts.Assert(cthd > 0);
var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, cthd);
var stats = new ParseStats(parent._host, cthd);
if (cthd <= 1)
{
consolidator = null;
return(new IRowCursor[1] {
new Cursor(parent, stats, active, reader, srcNeeded, 1)
});
}
consolidator = new Consolidator(cthd);
var cursors = new IRowCursor[cthd];
try
{
for (int i = 0; i < cursors.Length; i++)
{
cursors[i] = new Cursor(parent, stats, active, reader, srcNeeded, 1);
}
var result = cursors;
cursors = null;
return(result);
}
finally
{
if (cursors != null)
{
foreach (var curs in cursors)
{
if (curs != null)
{
curs.Dispose();
}
else
{
reader.Release();
stats.Release();
}
}
}
}
}
public static IRowCursor Create(TextLoader parent, IMultiStreamSource files, bool[] active)
{
// Note that files is allowed to be empty.
Contracts.AssertValue(parent);
Contracts.AssertValue(files);
Contracts.Assert(active == null || active.Length == parent._bindings.Infos.Length);
int srcNeeded;
int cthd;
SetupCursor(parent, active, 0, out srcNeeded, out cthd);
Contracts.Assert(cthd > 0);
var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, 1);
var stats = new ParseStats(parent._host, 1);
return(new Cursor(parent, stats, active, reader, srcNeeded, cthd));
}
// Note that we don't filter out rows with parsing issues since it's not acceptable to
// produce a different set of rows when subsetting columns. Any parsing errors need to be
// translated to NaN, not result in skipping the row. We should produce some diagnostics
// to alert the user to the issues.
private Cursor(TextLoader parent, ParseStats stats, bool[] active, LineReader reader, int srcNeeded, int cthd)
: base(parent._host)
{
Ch.Assert(active == null || active.Length == parent._bindings.Infos.Length);
Ch.AssertValue(reader);
Ch.AssertValue(stats);
Ch.Assert(srcNeeded >= 0);
Ch.Assert(cthd > 0);
_total = -1;
_batch = -1;
_bindings = parent._bindings;
_parser = parent._parser;
_active = active;
_reader = reader;
_stats = stats;
_srcNeeded = srcNeeded;
ParallelState state = null;
if (cthd > 1)
{
state = new ParallelState(this, out _rows, cthd);
}
else
{
_rows = _parser.CreateRowSet(_stats, 1, _active);
}
try
{
_getters = new Delegate[_bindings.Infos.Length];
for (int i = 0; i < _getters.Length; i++)
{
if (_active != null && !_active[i])
{
continue;
}
ColumnPipe v = _rows.Pipes[i];
Ch.Assert(v != null);
_getters[i] = v.GetGetter();
Ch.Assert(_getters[i] != null);
}
if (state != null)
{
_ator = ParseParallel(state).GetEnumerator();
state = null;
}
else
{
_ator = ParseSequential().GetEnumerator();
}
}
finally
{
if (state != null)
{
state.Dispose();
}
}
}