public override Token Next()
				{
					if (i == TOKENS.Length)
						return null;
					Token t = new Token(TOKENS[i], i, i);
					t.SetPositionIncrement(INCREMENTS[i]);
					i++;
					return t;
				}
Esempio n. 2
0
                public override Token Next()
                {
                    if (i == TOKENS.Length)
                    {
                        return(null);
                    }
                    Token t = new Token(TOKENS[i], i, i);

                    t.SetPositionIncrement(INCREMENTS[i]);
                    i++;
                    return(t);
                }
        /*
         * (non-Javadoc)
         *
         * @see Lucene.Net.Analysis.TokenStream#next()
         */
        public override Token Next(/* in */ Token reusableToken)
        {
            System.Diagnostics.Debug.Assert(reusableToken != null);
            int posIncr = 1;

            while (true)
            {
                int tokenType = scanner.GetNextToken();

                if (tokenType == StandardTokenizerImpl.YYEOF)
                {
                    return(null);
                }

                if (scanner.Yylength() <= maxTokenLength)
                {
                    reusableToken.Clear();
                    reusableToken.SetPositionIncrement(posIncr);
                    scanner.GetText(reusableToken);
                    int start = scanner.Yychar();
                    reusableToken.SetStartOffset(start);
                    reusableToken.SetEndOffset(start + reusableToken.TermLength());
                    // This 'if' should be removed in the next release. For now, it converts
                    // invalid acronyms to HOST. When removed, only the 'else' part should
                    // remain.
                    if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
                    {
                        if (replaceInvalidAcronym)
                        {
                            reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
                            reusableToken.SetTermLength(reusableToken.TermLength() - 1);                             // remove extra '.'
                        }
                        else
                        {
                            reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
                        }
                    }
                    else
                    {
                        reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
                    }
                    return(reusableToken);
                }
                // When we skip a too-long term, we still increment the
                // position increment
                else
                {
                    posIncr++;
                }
            }
        }
		/// <summary>Returns the next input Token, after being stemmed </summary>
        public override Token Next()
		{
			Token token = input.Next();
			if (token == null)
				return null;
			stemmer.SetCurrent(token.TermText());
			try
			{
                stemMethod(); 
			}
			catch (System.Exception e)
			{
				throw new System.SystemException(e.ToString());
			}
			
			Token newToken = new Token(stemmer.GetCurrent(), token.StartOffset(), token.EndOffset(), token.Type());
			newToken.SetPositionIncrement(token.GetPositionIncrement());
			return newToken;
		}
Esempio n. 5
0
        /// <summary>Returns the next input Token, after being stemmed </summary>
        public override Token Next()
        {
            Token token = input.Next();

            if (token == null)
            {
                return(null);
            }
            stemmer.SetCurrent(token.TermText());
            try
            {
                stemMethod.Invoke(stemmer, (System.Object[])EMPTY_ARGS);
            }
            catch (System.Exception e)
            {
                throw new System.SystemException(e.ToString());
            }

            Token newToken = new Token(stemmer.GetCurrent(), token.StartOffset(), token.EndOffset(), token.Type());

            newToken.SetPositionIncrement(token.GetPositionIncrement());
            return(newToken);
        }
			private void  InitBlock(HighlighterTest enclosingInstance)
			{
				this.enclosingInstance = enclosingInstance;
				lst = new System.Collections.ArrayList();
				Token t;
				t = new Token("hi", 0, 2);
				lst.Add(t);
				t = new Token("hispeed", 0, 8);
				lst.Add(t);
				t = new Token("speed", 3, 8);
				t.SetPositionIncrement(0);
				lst.Add(t);
				t = new Token("10", 8, 10);
				lst.Add(t);
				t = new Token("foo", 11, 14);
				lst.Add(t);
				iter = lst.GetEnumerator();
			}
		public override Token Next()
		{
			if (currentRealToken == null)
			{
				Token nextRealToken = realStream.Next();
				if (nextRealToken == null)
				{
					return null;
				}
				System.String expansions = (System.String) synonyms[nextRealToken.TermText()];
				if (expansions == null)
				{
					return nextRealToken;
				}
				st = new Tokenizer(expansions, ",");
				if (st.HasMoreTokens())
				{
					currentRealToken = nextRealToken;
				}
				return currentRealToken;
			}
			else
			{
				System.String nextExpandedValue = st.NextToken();
				Token expandedToken = new Token(nextExpandedValue, currentRealToken.StartOffset(), currentRealToken.EndOffset());
				expandedToken.SetPositionIncrement(0);
				if (!st.HasMoreTokens())
				{
					currentRealToken = null;
					st = null;
				}
				return expandedToken;
			}
		}
		/*
		* (non-Javadoc)
		*
		* @see Lucene.Net.Analysis.TokenStream#next()
		*/
		public override Token Next(Token result)
		{
			int posIncr = 1;
			
			while (true)
			{
				int tokenType = scanner.GetNextToken();
				
				if (tokenType == StandardTokenizerImpl.YYEOF)
				{
					return null;
				}
				
				if (scanner.Yylength() <= maxTokenLength)
				{
					result.Clear();
					result.SetPositionIncrement(posIncr);
					scanner.GetText(result);
					int start = scanner.Yychar();
					result.SetStartOffset(start);
					result.SetEndOffset(start + result.TermLength());
					// This 'if' should be removed in the next release. For now, it converts
					// invalid acronyms to HOST. When removed, only the 'else' part should
					// remain.
					if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
					{
						if (replaceInvalidAcronym)
						{
							result.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
							result.SetTermLength(result.TermLength() - 1); // remove extra '.'
						}
						else
						{
							result.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
						}
					}
					else
					{
						result.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
					}
					return result;
				}
				// When we skip a too-long term, we still increment the
				// position increment
				else
					posIncr++;
			}
		}
 public void TestNegativePositions()
 {
     SinkTokenizer tokens = new SinkTokenizer();
     Token t = new Token();
     t.SetTermText("a");
     t.SetPositionIncrement(0);
     tokens.Add(t);
     t.SetTermText("b");
     t.SetPositionIncrement(1);
     tokens.Add(t);
     t.SetTermText("c");
     tokens.Add(t);
     MockRAMDirectory dir = new MockRAMDirectory();
     IndexWriter w = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
     Document doc = new Document();
     doc.Add(new Field("field", tokens));
     w.AddDocument(doc);
     w.Close();
     IndexSearcher s = new IndexSearcher(dir);
     PhraseQuery pq = new PhraseQuery();
     pq.Add(new Term("field", "a"));
     pq.Add(new Term("field", "b"));
     pq.Add(new Term("field", "c"));
     Hits hits = s.Search(pq);
     Assert.AreEqual(1, hits.Length());
     Query q = new SpanTermQuery(new Term("field", "a"));
     hits = s.Search(q);
     Assert.AreEqual(1, hits.Length());
     TermPositions tps = s.GetIndexReader().TermPositions(new Term("field", "a"));
     Assert.IsTrue(tps.Next());
     Assert.AreEqual(1, tps.Freq());
     Assert.AreEqual(-1, tps.NextPosition());
     Assert.IsTrue(_TestUtil.CheckIndex(dir));
     s.Close();
     dir.Close();
 }
			public override Token Next()
			{
				if (tokenUpto >= Enclosing_Instance.tokens.Length)
					return null;
				else
				{
					Token t = new Token();
					TestToken testToken = Enclosing_Instance.tokens[tokenUpto++];
					t.SetTermText(testToken.text);
					if (tokenUpto > 1)
						t.SetPositionIncrement(testToken.pos - Enclosing_Instance.tokens[tokenUpto - 2].pos);
					else
						t.SetPositionIncrement(testToken.pos + 1);
					t.SetStartOffset(testToken.startOffset);
					t.SetEndOffset(testToken.endOffset);
					return t;
				}
			}