private void initTokenizer() { st = new StreamTokenizer(new StreamReader(i, Encoding.GetEncoding("ISO-8859-1"))); st.ResetSyntax(); st.WhitespaceChars(0, 32); st.WordChars(33, 255); st.CommentChar('!'); st.QuoteChar('\''); st.EolIsSignificant(false); }
private static StreamTokenizer CreateTokenizer(TextReader reader) { StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.ResetSyntax(); tokenizer.WordChars('a', 'z'); tokenizer.WordChars('A', 'Z'); tokenizer.WordChars(128 + 32, 255); tokenizer.WordChars('0', '9'); tokenizer.WordChars('-', '-'); tokenizer.WordChars('+', '+'); tokenizer.WordChars('.', '.'); tokenizer.WhitespaceChars(0, ' '); tokenizer.CommentChar('#'); return(tokenizer); }
/** * /// Constructs an ExtendedStreamTokenizer from the given Reader. This ExtendedStreamTokenizer has no comment * /// characters. * * /// @param reader the source of the data * /// @param eolIsSignificant true if eol is significant */ public ExtendedStreamTokenizer(StreamReader reader, Boolean eolIsSignificant) { _reader = reader; if (reader != null) { _tokenizer = new StreamTokenizer(reader); _tokenizer.ResetSyntax(); _tokenizer.WhitespaceChars(0, 32); _tokenizer.WordChars(33, 255); _tokenizer.EolIsSignificant(eolIsSignificant); } else { _atEof = true; } _putbackList = new List <String>(); }