public virtual void AddToken(AceToken t)
 {
     mTokens.Add(t);
 }
        /// <param name="filenamePrefix">path to an ACE .sgm file (but not including the .sgm extension)</param>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Xml.Sax.SAXException"/>
        /// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
        public static IList <IList <AceToken> > TokenizeAndSegmentSentences(string filenamePrefix)
        {
            IList <IList <AceToken> > sentences = new List <IList <AceToken> >();
            File   inputFile = new File(filenamePrefix + AceDocument.OrigExt);
            string input     = IOUtils.SlurpFile(inputFile);
            // now we can split the text into tokens
            RobustTokenizer <Word>            tokenizer = new RobustTokenizer <Word>(input);
            IList <RobustTokenizer.WordToken> tokenList = tokenizer.TokenizeToWordTokens();
            // and group the tokens into sentences
            List <AceToken> currentSentence = new List <AceToken>();
            int             quoteCount      = 0;

            for (int i = 0; i < tokenList.Count; i++)
            {
                RobustTokenizer.WordToken token = tokenList[i];
                string   tokenText      = token.GetWord();
                AceToken convertedToken = WordTokenToAceToken(token, sentences.Count);
                // start a new sentence if we skipped 2+ lines (after datelines, etc.)
                // or we hit some SGML
                // if (token.getNewLineCount() > 1 || AceToken.isSgml(tokenText)) {
                if (AceToken.IsSgml(tokenText))
                {
                    if (currentSentence.Count > 0)
                    {
                        sentences.Add(currentSentence);
                    }
                    currentSentence = new List <AceToken>();
                    quoteCount      = 0;
                }
                currentSentence.Add(convertedToken);
                if (tokenText.Equals("\""))
                {
                    quoteCount++;
                }
                // start a new sentence whenever we hit sentence-final punctuation
                if (sentenceFinalPuncSet.Contains(tokenText))
                {
                    // include quotes after EOS
                    if (i < tokenList.Count - 1 && quoteCount % 2 == 1 && tokenList[i + 1].GetWord().Equals("\""))
                    {
                        AceToken quoteToken = WordTokenToAceToken(tokenList[i + 1], sentences.Count);
                        currentSentence.Add(quoteToken);
                        quoteCount++;
                        i++;
                    }
                    if (currentSentence.Count > 0)
                    {
                        sentences.Add(currentSentence);
                    }
                    currentSentence = new List <AceToken>();
                    quoteCount      = 0;
                }
                else
                {
                    // start a new sentence when we hit an SGML tag
                    if (AceToken.IsSgml(tokenText))
                    {
                        if (currentSentence.Count > 0)
                        {
                            sentences.Add(currentSentence);
                        }
                        currentSentence = new List <AceToken>();
                        quoteCount      = 0;
                    }
                }
            }
            return(sentences);
        }