public ResampleCursor(ResampleTransform view, IRowCursor cursor, Func <int, bool> predicate, float lambda, int?seed, IRandom rand, Dictionary <UInt128, int> cache, int classColumn, TClass classValue) { _view = view; _inputCursor = cursor; _lambda = lambda; uint?useed = seed.HasValue ? (uint)seed.Value : (uint?)null; _rand = rand == null?RandomUtils.Create(useed) : rand; _predicate = predicate; _cache = cache; _maxReplica = cache == null?Math.Max((int)(lambda * 3 + 1), 1) : _cache.Values.Max(); int maxReplica = _maxReplica + 1; _shift = 0; _idGetter = cursor.GetIdGetter(); while (maxReplica > 0) { _shift += 1; maxReplica >>= 1; } _copy = -1; _classValue = classValue; _classGetter = classColumn >= 0 ? _inputCursor.GetGetter <TClass>(classColumn) : null; }
public Cursor(StatefulFilterTransform <TSrc, TDst, TState> parent, IRowCursor <TSrc> input, Func <int, bool> predicate) : base(parent.Host) { Ch.AssertValue(input); Ch.AssertValue(predicate); _parent = parent; _input = input; _src = new TSrc(); _dst = new TDst(); _state = new TState(); CursorChannelAttribute.TrySetCursorChannel(_parent.Host, _src, Ch); CursorChannelAttribute.TrySetCursorChannel(_parent.Host, _dst, Ch); CursorChannelAttribute.TrySetCursorChannel(_parent.Host, _state, Ch); if (parent._initStateAction != null) { parent._initStateAction(_state); } var appendedDataView = new DataViewConstructionUtils.SingleRowLoopDataView <TDst>(parent.Host, _parent._addedSchema); appendedDataView.SetCurrentRowObject(_dst); Func <int, bool> appendedPredicate = col => { col = _parent._bindings.AddedColumnIndices[col]; return(predicate(col)); }; _appendedRow = appendedDataView.GetRowCursor(appendedPredicate); }
public static Delegate GetGetterChoice <T1, T2>(IRowCursor cur, int col) { Delegate res = null; try { res = cur.GetGetter <T1>(col); if (res != null) { return(res); } } catch (Exception) { } try { res = cur.GetGetter <T2>(col); if (res != null) { return(res); } } catch (Exception) { } if (res == null) { throw Contracts.ExceptNotImpl($"Unable to get a getter for column {col} of type {typeof(T1)} or {typeof(T2)} from schema\n{SchemaHelper.ToString(cur.Schema)}."); } return(res); }
protected StatAggregatorBySlot(IChannel ch, ColumnType type, IRowCursor cursor, int col) : base(ch, cursor, col) { Ch.AssertValue(type); Ch.Assert(type.IsKnownSizeVector); Stat = new TStatItem[type.VectorSize]; }
protected RowCursorBase(RangeFilter parent, IRowCursor input, bool[] active) : base(parent.Host, input, parent.Schema, active) { Parent = parent; _min = Parent._min; _max = Parent._max; if (Parent._includeMin) { if (Parent._includeMax) { CheckBounds = Parent._complement ? (Func <Double, bool>)TestNotCC : TestCC; } else { CheckBounds = Parent._complement ? (Func <Double, bool>)TestNotCO : TestCO; } } else { if (Parent._includeMax) { CheckBounds = Parent._complement ? (Func <Double, bool>)TestNotOC : TestOC; } else { CheckBounds = Parent._complement ? (Func <Double, bool>)TestNotOO : TestOO; } } }
public sealed override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null) { Host.CheckValue(predicate, nameof(predicate)); Host.CheckValueOrNull(rand); var inputPred = _bindings.GetDependencies(predicate); var active = _bindings.GetActive(predicate); var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n, rand); Host.AssertNonEmpty(inputs); if (inputs.Length == 1 && n > 1 && WantParallelCursors(predicate)) { inputs = DataViewUtils.CreateSplitCursors(out consolidator, Host, inputs[0], n); } Host.AssertNonEmpty(inputs); var cursors = new IRowCursor[inputs.Length]; for (int i = 0; i < inputs.Length; i++) { cursors[i] = new RowCursor(Host, this, inputs[i], active); } return(cursors); }
public SubsetRowCursor(IRowCursor cursor, long start, long size) { this.cursor = cursor; this.start = start; end = start + (size - 1); MoveBeforeStart(); }
public RowCursor(IChannelProvider provider, int poolRows, IRowCursor input, IRandom rand) : base(provider) { Ch.AssertValue(input); Ch.AssertValue(rand); Ch.Assert(_blockSize > 0); Ch.Assert(_bufferDepth > 0); Ch.Assert(poolRows > 0); _poolRows = poolRows; _input = input; _rand = rand; _pipeIndices = Utils.GetIdentityPermutation(_poolRows - 1 + _bufferDepth * _blockSize); int colLim = Schema.ColumnCount; int numActive = 0; _colToActivesIndex = new int[colLim]; for (int c = 0; c < colLim; ++c) { _colToActivesIndex[c] = _input.IsColumnActive(c) ? numActive++ : -1; } _pipes = new ShufflePipe[numActive + (int)ExtraIndex.Lim]; _getters = new Delegate[numActive]; for (int c = 0; c < colLim; ++c) { int ia = _colToActivesIndex[c]; if (ia < 0) { continue; } _pipes[ia] = ShufflePipe.Create(_pipeIndices.Length, input.Schema.GetColumnType(c), RowCursorUtils.GetGetterAsDelegate(input, c)); _getters[ia] = CreateGetterDelegate(c); } var idPipe = _pipes[numActive + (int)ExtraIndex.Id] = ShufflePipe.Create(_pipeIndices.Length, NumberType.UG, input.GetIdGetter()); _idGetter = CreateGetterDelegate <UInt128>(idPipe); // Initially, after the preamble to MoveNextCore, we want: // liveCount=0, deadCount=0, circularIndex=0. So we set these // funky values accordingly. _pipeIndex = _circularIndex = _pipeIndices.Length - 1; _deadCount = -1; _liveCount = 1; // Set up the producer worker. _toConsume = new BufferBlock <int>(); _toProduce = new BufferBlock <int>(); // First request the pool - 1 + block size rows, to get us going. PostAssert(_toProduce, _poolRows - 1 + _blockSize); // Queue up the remaining capacity. for (int i = 1; i < _bufferDepth; ++i) { PostAssert(_toProduce, _blockSize); } _producerTask = LoopProducerWorker(); }
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null) { Host.CheckValue(predicate, nameof(predicate)); Host.CheckValueOrNull(rand); var bindings = GetBindings(); Func <int, bool> predicateInput; Func <int, bool> predicateMapper; var active = GetActive(bindings, predicate, out predicateInput, out predicateMapper); var inputs = Source.GetRowCursorSet(out consolidator, predicateInput, n, rand); Contracts.AssertNonEmpty(inputs); if (inputs.Length == 1 && n > 1 && WantParallelCursors(predicate) && (Source.GetRowCount() ?? int.MaxValue) > n) { inputs = DataViewUtils.CreateSplitCursors(out consolidator, Host, inputs[0], n); } Contracts.AssertNonEmpty(inputs); var cursors = new IRowCursor[inputs.Length]; for (int i = 0; i < inputs.Length; i++) { cursors[i] = new RowCursor(Host, this, inputs[i], active, predicateMapper); } return(cursors); }
public override IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null) { Host.CheckValue(predicate, nameof(predicate)); Host.CheckValueOrNull(rand); var inputPred = _bindings.GetDependencies(predicate); var active = _bindings.GetActive(predicate); IRowCursor input; if (n > 1 && ShouldUseParallelCursors(predicate) != false) { var inputs = Source.GetRowCursorSet(out consolidator, inputPred, n); Host.AssertNonEmpty(inputs); if (inputs.Length != 1) { var cursors = new IRowCursor[inputs.Length]; for (int i = 0; i < inputs.Length; i++) { cursors[i] = new RowCursor(Host, _bindings, inputs[i], active); } return(cursors); } input = inputs[0]; } else { input = Source.GetRowCursor(inputPred); } consolidator = null; return(new IRowCursor[] { new RowCursor(Host, _bindings, input, active) }); }
/// <summary> /// The base constructor class for the factory-based cursor creation. /// </summary> /// <param name="input"></param> /// <param name="signal">This method is called </param> protected TrainingCursorBase(IRowCursor input, Action <CursOpt> signal) { Contracts.AssertValue(input); Contracts.AssertValueOrNull(signal); _cursor = input; _signal = signal; }
public RowCursor(PerGroupTransformBase <TLabel, TScore, TState> parent, IRowCursor input, IRowCursor groupCursor, bool[] active) : base(parent.Host) { Ch.AssertValue(parent); // REVIEW: Try to see if we can relax this requirement in case there are no // active columns from the input (that are not required for output computation). Ch.AssertValue(input); Ch.AssertValue(groupCursor); Ch.AssertValue(active); _parent = parent; _input = input; _groupCursor = groupCursor; _active = active; _state = _parent.InitializeState(_input); var bindings = _parent.GetBindings(); _getters = _parent.CreateGetters(_state, iinfo => active[bindings.MapIinfoToCol(iinfo)]); _newGroupInGroupCursorDel = RowCursorUtils.GetIsNewGroupDelegate(_groupCursor, bindings.GroupIndex); _newGroupInInputCursorDel = RowCursorUtils.GetIsNewGroupDelegate(_input, bindings.GroupIndex); _labelGetter = _parent.GetLabelGetter(_groupCursor); _scoreGetter = _parent.GetScoreGetter(_groupCursor); }
public AddRandomCursor(AddRandomTransform view, IRowCursor cursor) { _view = view; _inputCursor = cursor; _schema = _view.Schema; _rand = new Random(_view._args.seed); }
public ISlotCursor GetSlotCursor(int col) { _host.CheckParam(0 <= col && col < _header.ColumnCount, nameof(col)); var view = _entries[col].GetViewOrNull(); if (view == null) { throw _host.ExceptParam(nameof(col), "Bad call to GetSlotCursor on untransposable column '{0}'", Schema.GetColumnName(col)); } _host.CheckParam(0 <= col && col < _header.ColumnCount, nameof(col)); // We don't want the type error, if there is one, to be handled by the get-getter, because // at the point we've gotten the interior cursor, but not yet constructed the slot cursor. ColumnType cursorType = TransposeSchema.GetSlotType(col).ItemType; IRowCursor inputCursor = view.GetRowCursor(c => true); try { return(Utils.MarshalInvoke(GetSlotCursorCore <int>, cursorType.RawType, inputCursor)); } catch (Exception) { // We've already verified the types so we shouldn't throw here, in principle, but just // be extra careful so we're sure to dispose the input cursor. if (inputCursor != null) { inputCursor.Dispose(); } throw; } }
public SlotCursor(TransposeLoader parent, IRowCursor cursor) : base(parent._host, cursor) { _parent = parent; Ch.Assert(cursor.Schema.ColumnCount == 1); Ch.Assert(cursor.Schema.GetColumnType(0).RawType == typeof(VBuffer <T>)); _getter = Input.GetGetter <VBuffer <T> >(0); }
public Delegate[] GetCursorGetter(IRowCursor cursor) { return(new Delegate[] { cursor.GetGetter <float[]>(0), cursor.GetGetter <uint>(1), }); }
public DBScanCursor(DBScanState view, IRowCursor cursor) { _view = view; _colCluster = view.Source.Schema.ColumnCount; _colScore = _colCluster + 1; _colName = _colScore + 1; _inputCursor = cursor; }
public CursorType(TemporaryViewCursorColumn <TRepValue> view, Func <int, bool> needCol, IRowCursor otherValues) { _needCol = needCol; _view = view; _state = CursorState.NotStarted; _otherValues = otherValues; _ignoreOtherColumn = view._ignoreOtherColumn; }
public static IRowCursor CreateKeyRowCursor(RangeFilter filter, IRowCursor input, bool[] active) { Contracts.Assert(filter._type.IsKey); Func <RangeFilter, IRowCursor, bool[], IRowCursor> del = CreateKeyRowCursor <int>; var methodInfo = del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(filter._type.RawType); return((IRowCursor)methodInfo.Invoke(null, new object[] { filter, input, active })); }
/// <summary> /// Constructor. /// </summary> /// <param name="column">column to be replaced</param> /// <param name="schema">schema of the view</param> /// <param name="otherValues">cursor which contains the others values</param> /// <param name="ignoreOtherColumn">ignore other column if they are being requested</param> public InfiniteLoopViewCursorColumn(int column, Schema schema = null, IRowCursor otherValues = null, bool ignoreOtherColumn = false) { _column = column; _otherValues = otherValues; _schema = otherValues == null ? schema : otherValues.Schema; _ownCursor = null; _ignoreOtherColumn = ignoreOtherColumn; Contracts.AssertValue(_schema); }
internal static List <ConfusionMatrix> Create(IHostEnvironment env, IDataView confusionMatrix) { Contracts.AssertValue(env); env.AssertValue(confusionMatrix); if (!confusionMatrix.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.Count, out int countColumn)) { throw env.Except($"ConfusionMatrix data view did not contain a {nameof(MetricKinds.ColumnNames.Count)} column."); } IRowCursor cursor = confusionMatrix.GetRowCursor(col => col == countColumn); var slots = default(VBuffer <ReadOnlyMemory <char> >); confusionMatrix.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, countColumn, ref slots); var slotsValues = slots.GetValues(); string[] classNames = new string[slotsValues.Length]; for (int i = 0; i < slotsValues.Length; i++) { classNames[i] = slotsValues[i].ToString(); } ColumnType type = confusionMatrix.Schema.GetColumnType(countColumn); env.Assert(type.IsVector); ValueGetter <VBuffer <double> > countGetter = cursor.GetGetter <VBuffer <double> >(countColumn); VBuffer <double> countValues = default; List <ConfusionMatrix> confusionMatrices = new List <ConfusionMatrix>(); int valuesRowIndex = 0; double[,] elements = null; while (cursor.MoveNext()) { if (valuesRowIndex == 0) { elements = new double[type.VectorSize, type.VectorSize]; } countGetter(ref countValues); ReadOnlySpan <double> values = countValues.GetValues(); for (int i = 0; i < values.Length; i++) { elements[valuesRowIndex, i] = values[i]; } valuesRowIndex++; if (valuesRowIndex == type.VectorSize) { valuesRowIndex = 0; confusionMatrices.Add(new ConfusionMatrix(elements, classNames)); } } return(confusionMatrices); }
public UpdatableResultSetView(SystemTransaction transaction, IMutableTable backedTable, Expression[] project, IRowCursor select) { this.transaction = transaction; this.backedTable = backedTable; this.project = project; originalSelect = select; currentSelect = select; this.select = null; }
public static IRowCursor[] CreateSet(out IRowCursorConsolidator consolidator, TextLoader parent, IMultiStreamSource files, bool[] active, int n) { // Note that files is allowed to be empty. Contracts.AssertValue(parent); Contracts.AssertValue(files); Contracts.Assert(active == null || active.Length == parent._bindings.Infos.Length); int srcNeeded; int cthd; SetupCursor(parent, active, n, out srcNeeded, out cthd); Contracts.Assert(cthd > 0); var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, cthd); var stats = new ParseStats(parent._host, cthd); if (cthd <= 1) { consolidator = null; return(new IRowCursor[1] { new Cursor(parent, stats, active, reader, srcNeeded, 1) }); } consolidator = new Consolidator(cthd); var cursors = new IRowCursor[cthd]; try { for (int i = 0; i < cursors.Length; i++) { cursors[i] = new Cursor(parent, stats, active, reader, srcNeeded, 1); } var result = cursors; cursors = null; return(result); } finally { if (cursors != null) { foreach (var curs in cursors) { if (curs != null) { curs.Dispose(); } else { reader.Release(); stats.Release(); } } } } }
protected FloatLabelCursor(IRowCursor input, RoleMappedData data, CursOpt opt, Action <CursOpt> signal = null) : base(input, data, opt, signal) { if ((opt & CursOpt.Label) != 0 && data.Schema.Label != null) { _get = Row.GetLabelFloatGetter(data); _keepBad = (opt & CursOpt.AllowBadLabels) != 0; } }
public PrefetchRowCursor(IRowCursor cursor, ITable table) { this.cursor = cursor; this.table = table; for (int i = 0; i < pageRead.Length; i++) { pageRead[i] = -1; } }
public OpticsOrderingCursor(OpticsOrderingState view, IRowCursor cursor) { _view = view; _colOrdering = view.Source.Schema.ColumnCount; _colReachability = _colOrdering + 1; _colCore = _colReachability + 1; _colName = _colCore + 1; _inputCursor = cursor; }
public RowCursor(IChannelProvider provider, IRowCursor input, Schema schema, bool[] active, long skip, long take) : base(provider, input, schema, active) { Ch.Assert(skip >= 0); Ch.Assert(take >= 0); _skip = skip; _take = take; }
/// <summary> /// Constructor. /// </summary> /// <param name="value">value which will replace the value</param> /// <param name="column">column to be replaced</param> /// <param name="otherValues">cursor which contains the others values</param> /// <param name="schema">schema to replace if otherValues is null</param> public TemporaryViewCursorColumn(TRepValue value, int column, Schema schema = null, IRowCursor otherValues = null, bool ignoreOtherColumn = false) { _column = column; _otherValues = otherValues; _schema = otherValues == null ? schema : otherValues.Schema; _value = value; _ignoreOtherColumn = ignoreOtherColumn; Contracts.AssertValue(_schema); }
protected FeatureFloatVectorCursor(IRowCursor input, RoleMappedData data, CursOpt opt, Action <CursOpt> signal = null) : base(input, data, opt, signal) { if ((opt & CursOpt.Features) != 0 && data.Schema.Feature != null) { _get = Row.GetFeatureFloatVectorGetter(data); _keepBad = (opt & CursOpt.AllowBadFeatures) != 0; } }
public RowCursor(NAFilter parent, IRowCursor input, bool[] active) : base(parent.Host, input, parent.Schema, active) { _parent = parent; _values = new Value[_parent._infos.Length]; for (int i = 0; i < _parent._infos.Length; i++) { _values[i] = Value.Create(this, _parent._infos[i]); } }
public ScalerCursor(IRowCursor cursor, ScalerTransform parent, Func <int, bool> predicate) { _inputCursor = cursor; _parent = parent; _scalingFactors = parent._scalingFactors; if (_scalingFactors == null) { throw parent._host.ExceptValue("The transform was never trained. Predictions cannot be computed."); } }
public MinMaxAggregatorBySlot(IChannel ch, ColumnType type, IRowCursor cursor, int col, bool returnMax) : base(ch, type, cursor, col, returnMax) { Single bound = ReturnMax ? Single.NegativeInfinity : Single.PositiveInfinity; for (int i = 0; i < Stat.Length; i++) { Stat[i] = bound; } }
public MinMaxAggregatorBySlot(IChannel ch, ColumnType type, IRowCursor cursor, int col, bool returnMax) : base(ch, type, cursor, col, returnMax) { Double bound = ReturnMax ? Double.MinValue : Double.MaxValue; for (int i = 0; i < Stat.Length; i++) { Stat[i] = bound; } }
public SelectedRowCursor(IRowCursor rowCursor) : base(rowCursor) { }
public IRowCursor GetCurrentRowCursor() { if (currentSelect == null) currentSelect = new DefaultRowCursor(GetCurrentIndex().GetCursor()); return currentSelect; }
public void Finish(bool commit) { if (updateType == ' ') throw new InvalidOperationException("No update operations"); try { // We get the index before we complete the operation. This ensures // the original iterator will not be corrupted by the complete // operation. // Get the index (this may need to copy the index from the view). IIndex<RowId> index = null; if (commit) { index = GetCurrentIndex(); } // If we are to do the operation, if (commit) { // Get the native table name being updated, TableName backedTname = BackedTableName; // Perform the operation via the SQLIterpreter. The interpreter // checks for immediate referential constraint violations and // updates any indexes on the table. // Note that these operations may change the state of // 'original_iterator' if (updateType == 'U') { // Update indexes, check integrity, etc // This may generate an exception and fail the operation. if (backedTname != null) { SqlInterpreter.CompleteRowUpdate(transaction, backedTable, backedTname, updatedRow.Id, currentRow.Id); } // Remove the value at the old position and insert a new value index.RemoveAt(oldRowIndex); index.Insert(currentRow.Id, oldRowIndex); backedTable.Update(currentRow); } else if (updateType == 'I') { // Update indexes, check integrity, etc // This may generate an exception and fail the operation. if (backedTname != null) { SqlInterpreter.CompleteRowInsert(transaction, backedTable, backedTname, currentRow.Id); } // Insert the row identifier on the end of the index index.Add(currentRow.Id); backedTable.Insert(currentRow); } else if (updateType == 'R') { // Update indexes, check integrity, etc // This may generate an exception and fail the operation. if (backedTname != null) { SqlInterpreter.CompleteRowRemove(transaction, backedTable, backedTname, currentRow.Id); } // Remove the entry from the index index.RemoveAt(oldRowIndex); } else { // This should never be able to happen throw new ApplicationException("Unexpected update type: " + updateType); } // Invalidate the current select iterator. currentSelect = null; } else { // Fail the operation backedTable.Undo(); } } finally { // Ensure we invalidate all this state information updateType = ' '; oldRowIndex = -1; currentRow = null; } }
public ReverseRowCursor(IRowCursor cursor) : this(cursor, cursor.Count) { }
public ReverseRowCursor(IRowCursor cursor, long size) { this.cursor = cursor; this.size = size; MoveBeforeStart(); }
internal SystemRowCursor(IRowCursor rowCursor) { this.rowCursor = rowCursor; }
public void InitGroups(QueryProcessor processor, IIndex<long> emptyIndexContainer) { Debug.Assert(emptyIndexContainer.Count == 0); ITable child = BaseTable; // No groups, so make the entire child table the group, if (aggregateComposite == null || child.RowCount <= 1) { emptyIndexContainer.Add(0); emptyIndexContainer.Add(child.RowCount); } // Populate the index by the aggregate composite, else { // Create a resolver for the composite function IndexResolver resolver = processor.CreateResolver(child, aggregateComposite); // The groups state long groupPos = 0; long groupSize = 0; SqlObject[] lastComposite = null; // Scan over the child IRowCursor cursor = child.GetRowCursor(); while (cursor.MoveNext()) { RowId rowid = cursor.Current; // Get the group term SqlObject[] groupValue = resolver.GetValue(rowid); if (lastComposite == null) { lastComposite = groupValue; } else { int c = SqlObject.Compare(groupValue, lastComposite); // If group_val > the last composite, we are on a new group if (c > 0) { // New group, emptyIndexContainer.Add(groupPos); emptyIndexContainer.Add(groupSize); lastComposite = groupValue; groupPos = groupPos + groupSize; groupSize = 0; } else if (c < 0) { // This will happen if the child table is not sorted by the // composite expression. throw new ApplicationException("Aggregate child is not sorted correctly."); } } ++groupSize; } // Final group // (the below check probably not necessary since we already check for the // empty child so group size will always be >1 at this point). if (groupSize > 0) { emptyIndexContainer.Add(groupPos); emptyIndexContainer.Add(groupSize); } } // Set the group index childGroupsIndex = emptyIndexContainer; lookupCursor = BaseTable.GetRowCursor(); }
public SubsetTable(ITable child, IRowCursor subset) : base(child) { this.subset = subset; indexRequestsFallthrough = false; }
public void Dispose() { cursor.Dispose(); cursor = null; }
public bool MoveBack() { currentCursor = cursors[cursorIndex]; if (!currentCursor.MoveBack()) { // Set to after the end of the previous cursor in the set currentCursor = cursors[--cursorIndex]; cursorOffset = currentCursor.Count; currentCursor.MoveTo(cursorOffset); } return --position > 0; }
public bool MoveNext() { currentCursor = cursors[cursorIndex]; if (!currentCursor.MoveNext()) { // Set to before the first of the next cursor in the set currentCursor = cursors[++cursorIndex]; cursorOffset = -1; currentCursor.MoveTo(cursorOffset); } return ++position < Count; }