public Row[] Next() { if (!HasNext()) throw new Exception("no more elements"); var rows = new Row[_columnRowCounters.Length]; for (int i = 0; i < _columnRowCounters.Length; i++) { rows[i] = _enclosingInstance.Columns[i].Rows[_columnRowCounters[i]]; } IncrementColumnRowCounters(); return rows; }
/// <summary> /// Loads one column from the token stream. /// /// When the last token is read from the token stream it will column.setLast(true); /// </summary> /// <returns>true if it manage to read one more column from the input token stream</returns> private bool ReadColumn() { Token token; if (_readColumnBuf != null) { token = _readColumnBuf; _readColumnBuf = null; } else { token = GetNextInputToken(new Token()); } if (token == null) return false; var currentReaderColumn = new Column(Matrix); var currentReaderRow = new Row(currentReaderColumn); currentReaderRow.Tokens.AddLast(token); TokenPositioner tokenPositioner; while ((_readColumnBuf = GetNextInputToken(new Token())) != null && (tokenPositioner = _settingsCodec.GetTokenPositioner(_readColumnBuf)) != TokenPositioner.NewColumn) { if (tokenPositioner == TokenPositioner.SameRow) { currentReaderRow.Tokens.AddLast(_readColumnBuf); } else { currentReaderRow = new Row(currentReaderColumn); currentReaderRow.Tokens.AddLast(_readColumnBuf); } _readColumnBuf = null; } if (_readColumnBuf == null) { _readColumnBuf = GetNextInputToken(new Token()); if (_readColumnBuf == null) currentReaderColumn.IsLast = true; } return true; }
public Column(Token token, Analysis.Shingle.Matrix.Matrix enclosingInstance) : this(enclosingInstance) { var row = new Row(this); row.Tokens.AddLast(token); }
public void TestMatrix() { // some other tests set this to null. // set it here in case tests are run out of the usual order. ShingleMatrixFilter.DefaultSettingsCodec = new SimpleThreeDimensionalTokenSettingsCodec(); var matrix = new Matrix.Matrix(); new Column(TokenFactory("no", 1), matrix); new Column(TokenFactory("surprise", 1), matrix); new Column(TokenFactory("to", 1), matrix); new Column(TokenFactory("see", 1), matrix); new Column(TokenFactory("england", 1), matrix); new Column(TokenFactory("manager", 1), matrix); var col = new Column(matrix); // sven göran eriksson is a multi token synonym to svennis new Row(col).Tokens.AddLast(TokenFactory("svennis", 1)); var row = new Row(col); row.Tokens.AddLast(TokenFactory("sven", 1)); row.Tokens.AddLast(TokenFactory("göran", 1)); row.Tokens.AddLast(TokenFactory("eriksson", 1)); new Column(TokenFactory("in", 1), matrix); new Column(TokenFactory("the", 1), matrix); new Column(TokenFactory("croud", 1), matrix); TokenStream ts = new ShingleMatrixFilter(matrix, 2, 4, '_', true, new SimpleThreeDimensionalTokenSettingsCodec()); // for (Token token = ts.next(new Token()); token != null; token = ts.next(token)) { // System.out.println("assertNext(ts, \"" + token.term() + "\", " + token.getPositionIncrement() + ", " + (token.getPayload() == null ? "1.0" : PayloadHelper.decodeFloat(token.getPayload().getData())) + "f, " + token.startOffset() + ", " + token.endOffset() + ");"); // token.clear(); // } AssertNext(ts, "no_surprise", 1, 1.4142135f, 0, 0); AssertNext(ts, "no_surprise_to", 1, 1.7320508f, 0, 0); AssertNext(ts, "no_surprise_to_see", 1, 2.0f, 0, 0); AssertNext(ts, "surprise_to", 1, 1.4142135f, 0, 0); AssertNext(ts, "surprise_to_see", 1, 1.7320508f, 0, 0); AssertNext(ts, "surprise_to_see_england", 1, 2.0f, 0, 0); AssertNext(ts, "to_see", 1, 1.4142135f, 0, 0); AssertNext(ts, "to_see_england", 1, 1.7320508f, 0, 0); AssertNext(ts, "to_see_england_manager", 1, 2.0f, 0, 0); AssertNext(ts, "see_england", 1, 1.4142135f, 0, 0); AssertNext(ts, "see_england_manager", 1, 1.7320508f, 0, 0); AssertNext(ts, "see_england_manager_svennis", 1, 2.0f, 0, 0); AssertNext(ts, "england_manager", 1, 1.4142135f, 0, 0); AssertNext(ts, "england_manager_svennis", 1, 1.7320508f, 0, 0); AssertNext(ts, "england_manager_svennis_in", 1, 2.0f, 0, 0); AssertNext(ts, "manager_svennis", 1, 1.4142135f, 0, 0); AssertNext(ts, "manager_svennis_in", 1, 1.7320508f, 0, 0); AssertNext(ts, "manager_svennis_in_the", 1, 2.0f, 0, 0); AssertNext(ts, "svennis_in", 1, 1.4142135f, 0, 0); AssertNext(ts, "svennis_in_the", 1, 1.7320508f, 0, 0); AssertNext(ts, "svennis_in_the_croud", 1, 2.0f, 0, 0); AssertNext(ts, "in_the", 1, 1.4142135f, 0, 0); AssertNext(ts, "in_the_croud", 1, 1.7320508f, 0, 0); AssertNext(ts, "the_croud", 1, 1.4142135f, 0, 0); AssertNext(ts, "see_england_manager_sven", 1, 2.0f, 0, 0); AssertNext(ts, "england_manager_sven", 1, 1.7320508f, 0, 0); AssertNext(ts, "england_manager_sven_göran", 1, 2.0f, 0, 0); AssertNext(ts, "manager_sven", 1, 1.4142135f, 0, 0); AssertNext(ts, "manager_sven_göran", 1, 1.7320508f, 0, 0); AssertNext(ts, "manager_sven_göran_eriksson", 1, 2.0f, 0, 0); AssertNext(ts, "sven_göran", 1, 1.4142135f, 0, 0); AssertNext(ts, "sven_göran_eriksson", 1, 1.7320508f, 0, 0); AssertNext(ts, "sven_göran_eriksson_in", 1, 2.0f, 0, 0); AssertNext(ts, "göran_eriksson", 1, 1.4142135f, 0, 0); AssertNext(ts, "göran_eriksson_in", 1, 1.7320508f, 0, 0); AssertNext(ts, "göran_eriksson_in_the", 1, 2.0f, 0, 0); AssertNext(ts, "eriksson_in", 1, 1.4142135f, 0, 0); AssertNext(ts, "eriksson_in_the", 1, 1.7320508f, 0, 0); AssertNext(ts, "eriksson_in_the_croud", 1, 2.0f, 0, 0); Assert.IsFalse(ts.IncrementToken()); }