public static Token Next(StandardTokenizerImpl scanner)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerImpl.YYEOF)
{
return(null);
}
int startPosition = scanner.yychar();
string tokenImage = scanner.yytext();
return(new Token(tokenImage, startPosition, startPosition
+ tokenImage.Length,
StandardTokenizerImpl.TOKEN_TYPES[tokenType]));
}
public const int ACRONYM_DEP = 8; /* deprecated */
public static void Main(string[] args)
{
StringBuilder sb = new StringBuilder();
foreach (string arg in args)
{
sb.Append(arg + " ");
}
StringReader reader = new StringReader(sb.ToString());
StandardTokenizerImpl scanner = StandardTokenizerImpl.GetStandardTokenizerImpl(reader);
Lucene.Net.Analysis.Token token;
while ((token = Next(scanner)) != null)
{
Console.WriteLine("{0}", token.TermText());
}
}