private void initTokenizer()
{
st = new StreamTokenizer(new StreamReader(i, Encoding.GetEncoding("ISO-8859-1")));
st.ResetSyntax();
st.WhitespaceChars(0, 32);
st.WordChars(33, 255);
st.CommentChar('!');
st.QuoteChar('\'');
st.EolIsSignificant(false);
}
/**
* /// Constructs an ExtendedStreamTokenizer from the given Reader. This ExtendedStreamTokenizer has no comment
* /// characters.
*
* /// @param reader the source of the data
* /// @param eolIsSignificant true if eol is significant
*/
public ExtendedStreamTokenizer(StreamReader reader, Boolean eolIsSignificant)
{
_reader = reader;
if (reader != null)
{
_tokenizer = new StreamTokenizer(reader);
_tokenizer.ResetSyntax();
_tokenizer.WhitespaceChars(0, 32);
_tokenizer.WordChars(33, 255);
_tokenizer.EolIsSignificant(eolIsSignificant);
}
else
{
_atEof = true;
}
_putbackList = new List <String>();
}