/*
         * (non-Javadoc)
         *
         * @see Lucene.Net.Analysis.TokenStream#next()
         */
        public override Token Next(/* in */ Token reusableToken)
        {
            System.Diagnostics.Debug.Assert(reusableToken != null);
            int posIncr = 1;

            while (true)
            {
                int tokenType = scanner.GetNextToken();

                if (tokenType == StandardTokenizerImpl.YYEOF)
                {
                    return(null);
                }

                if (scanner.Yylength() <= maxTokenLength)
                {
                    reusableToken.Clear();
                    reusableToken.SetPositionIncrement(posIncr);
                    scanner.GetText(reusableToken);
                    int start = scanner.Yychar();
                    reusableToken.SetStartOffset(start);
                    reusableToken.SetEndOffset(start + reusableToken.TermLength());
                    // This 'if' should be removed in the next release. For now, it converts
                    // invalid acronyms to HOST. When removed, only the 'else' part should
                    // remain.
                    if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
                    {
                        if (replaceInvalidAcronym)
                        {
                            reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
                            reusableToken.SetTermLength(reusableToken.TermLength() - 1);                             // remove extra '.'
                        }
                        else
                        {
                            reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
                        }
                    }
                    else
                    {
                        reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
                    }
                    return(reusableToken);
                }
                // When we skip a too-long term, we still increment the
                // position increment
                else
                {
                    posIncr++;
                }
            }
        }
		/*
		* (non-Javadoc)
		*
		* @see Lucene.Net.Analysis.TokenStream#next()
		*/
		public override Token Next(Token result)
		{
			int posIncr = 1;
			
			while (true)
			{
				int tokenType = scanner.GetNextToken();
				
				if (tokenType == StandardTokenizerImpl.YYEOF)
				{
					return null;
				}
				
				if (scanner.Yylength() <= maxTokenLength)
				{
					result.Clear();
					result.SetPositionIncrement(posIncr);
					scanner.GetText(result);
					int start = scanner.Yychar();
					result.SetStartOffset(start);
					result.SetEndOffset(start + result.TermLength());
					// This 'if' should be removed in the next release. For now, it converts
					// invalid acronyms to HOST. When removed, only the 'else' part should
					// remain.
					if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
					{
						if (replaceInvalidAcronym)
						{
							result.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
							result.SetTermLength(result.TermLength() - 1); // remove extra '.'
						}
						else
						{
							result.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
						}
					}
					else
					{
						result.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
					}
					return result;
				}
				// When we skip a too-long term, we still increment the
				// position increment
				else
					posIncr++;
			}
		}
			public override Token Next()
			{
				if (tokenUpto >= Enclosing_Instance.tokens.Length)
					return null;
				else
				{
					Token t = new Token();
					TestToken testToken = Enclosing_Instance.tokens[tokenUpto++];
					t.SetTermText(testToken.text);
					if (tokenUpto > 1)
						t.SetPositionIncrement(testToken.pos - Enclosing_Instance.tokens[tokenUpto - 2].pos);
					else
						t.SetPositionIncrement(testToken.pos + 1);
					t.SetStartOffset(testToken.startOffset);
					t.SetEndOffset(testToken.endOffset);
					return t;
				}
			}