private IEnumerable <int> ParseSequential()
{
Ch.AssertValue(_rows);
Ch.Assert(_rows.Count == 1);
LineBatch batch;
var helper = _parser.CreateHelper(_rows.Stats, _srcNeeded);
while ((batch = _reader.GetBatch()).Infos != null)
{
Ch.Assert(batch.Exception == null);
_total = batch.Total;
foreach (var info in batch.Infos)
{
Ch.Assert(info.Line > 0);
Ch.AssertNonEmpty(info.Text);
_parser.ParseRow(_rows, 0, helper, _active, batch.Path, info.Line, info.Text);
_batch = batch.Batch;
yield return(0);
++_total;
}
_batch = long.MaxValue;
}
}
public RowCursor(NltTokenizeTransform parent, IRowCursor input, bool[] active)
: base(parent.Host, input)
{
Ch.AssertValue(parent._bindings);
Ch.AssertNonEmpty(parent._bindings.Infos);
Ch.AssertNonEmpty(parent._bindings.GroupInfos);
Ch.Assert(active == null || active.Length == parent._bindings.ColumnCount);
_bindings = parent._bindings;
_active = active;
var resourcesExist = new bool?[Tokenizers.Length];
var infos = parent._bindings.Infos;
_getters = new ValueGetter<VBuffer<DvText>>[infos.Length];
var groupCurHelpers = new GroupCursorHelper[_bindings.SrcColsCount];
for (int iinfo = 0; iinfo < infos.Length; iinfo++)
{
if (_active != null && !_active[_bindings.MapIinfoToCol(iinfo)])
continue;
int groupId = infos[iinfo].GroupInfoId;
if (groupCurHelpers[groupId] == null)
groupCurHelpers[groupId] = new GroupCursorHelper(input, _bindings.GroupInfos[groupId], resourcesExist);
var groupState = groupCurHelpers[groupId];
if (infos[iinfo].IsTypes)
_getters[iinfo] = groupState.GetTypes;
else
_getters[iinfo] = groupState.GetTokens;
}
}
public CursorBase(AppendRowsDataView parent)
: base(parent._host)
{
Sources = parent._sources;
Ch.AssertNonEmpty(Sources);
Schema = parent._schema;
Getters = new Delegate[Schema.Count];
}
public Cursor(ZipDataView parent, DataViewRowCursor[] srcCursors, Func <int, bool> predicate)
: base(parent._host)
{
Ch.AssertNonEmpty(srcCursors);
Ch.AssertValue(predicate);
_cursors = srcCursors;
_zipBinding = parent._zipBinding;
_isColumnActive = Utils.BuildArray(_zipBinding.ColumnCount, predicate);
}
public Cursor(ZipDataView parent, IRowCursor[] srcCursors, Func <int, bool> predicate)
: base(parent._host)
{
Ch.AssertNonEmpty(srcCursors);
Ch.AssertValue(predicate);
_cursors = srcCursors;
_compositeSchema = parent._compositeSchema;
_isColumnActive = Utils.BuildArray(_compositeSchema.ColumnCount, predicate);
}
