private void initTokenizer() { st = new StreamTokenizer(new StreamReader(i, Encoding.GetEncoding("ISO-8859-1"))); st.ResetSyntax(); st.WhitespaceChars(0, 32); st.WordChars(33, 255); st.CommentChar('!'); st.QuoteChar('\''); st.EolIsSignificant(false); }
/** * /// Constructs an ExtendedStreamTokenizer from the given Reader. This ExtendedStreamTokenizer has no comment * /// characters. * * /// @param reader the source of the data * /// @param eolIsSignificant true if eol is significant */ public ExtendedStreamTokenizer(StreamReader reader, Boolean eolIsSignificant) { _reader = reader; if (reader != null) { _tokenizer = new StreamTokenizer(reader); _tokenizer.ResetSyntax(); _tokenizer.WhitespaceChars(0, 32); _tokenizer.WordChars(33, 255); _tokenizer.EolIsSignificant(eolIsSignificant); } else { _atEof = true; } _putbackList = new List <String>(); }