protected internal override T GetNext() { try { T nextToken = null; do { // Depending on the orthographic normalization options, // some tokens can be obliterated. In this case, keep iterating // until we see a non-zero length token. nextToken = ((splitContractions || splitCompounds) && compoundBuffer.Count > 0) ? (T)compoundBuffer.Remove(0) : (T)lexer.Next(); }while (nextToken != null && nextToken.Word().Length == 0); // Check for compounds to split if (splitCompounds && nextToken is CoreLabel) { CoreLabel cl = (CoreLabel)nextToken; if (cl.ContainsKey(typeof(CoreAnnotations.ParentAnnotation)) && cl.Get(typeof(CoreAnnotations.ParentAnnotation)).Equals(FrenchLexer.CompoundAnnotation)) { nextToken = (T)ProcessCompound(cl); } } // Check for contractions to split if (splitContractions && nextToken is CoreLabel) { CoreLabel cl = (CoreLabel)nextToken; if (cl.ContainsKey(typeof(CoreAnnotations.ParentAnnotation)) && cl.Get(typeof(CoreAnnotations.ParentAnnotation)).Equals(FrenchLexer.ContrAnnotation)) { nextToken = (T)ProcessContraction(cl); } } return(nextToken); } catch (IOException e) { throw new RuntimeIOException(e); } }