protected internal override T GetNext()
{
try
{
T nextToken = null;
do
{
// Depending on the orthographic normalization options,
// some tokens can be obliterated. In this case, keep iterating
// until we see a non-zero length token.
nextToken = ((splitContractions || splitCompounds) && compoundBuffer.Count > 0) ? (T)compoundBuffer.Remove(0) : (T)lexer.Next();
}while (nextToken != null && nextToken.Word().Length == 0);
// Check for compounds to split
if (splitCompounds && nextToken is CoreLabel)
{
CoreLabel cl = (CoreLabel)nextToken;
if (cl.ContainsKey(typeof(CoreAnnotations.ParentAnnotation)) && cl.Get(typeof(CoreAnnotations.ParentAnnotation)).Equals(FrenchLexer.CompoundAnnotation))
{
nextToken = (T)ProcessCompound(cl);
}
}
// Check for contractions to split
if (splitContractions && nextToken is CoreLabel)
{
CoreLabel cl = (CoreLabel)nextToken;
if (cl.ContainsKey(typeof(CoreAnnotations.ParentAnnotation)) && cl.Get(typeof(CoreAnnotations.ParentAnnotation)).Equals(FrenchLexer.ContrAnnotation))
{
nextToken = (T)ProcessContraction(cl);
}
}
return(nextToken);
}
catch (IOException e)
{
throw new RuntimeIOException(e);
}
}
