private void initTokenizer()
{
st = new StreamTokenizer(new StreamReader(i, Encoding.GetEncoding("ISO-8859-1")));
st.ResetSyntax();
st.WhitespaceChars(0, 32);
st.WordChars(33, 255);
st.CommentChar('!');
st.QuoteChar('\'');
st.EolIsSignificant(false);
}
private static StreamTokenizer CreateTokenizer(TextReader reader)
{
StreamTokenizer tokenizer = new StreamTokenizer(reader);
tokenizer.ResetSyntax();
tokenizer.WordChars('a', 'z');
tokenizer.WordChars('A', 'Z');
tokenizer.WordChars(128 + 32, 255);
tokenizer.WordChars('0', '9');
tokenizer.WordChars('-', '-');
tokenizer.WordChars('+', '+');
tokenizer.WordChars('.', '.');
tokenizer.WhitespaceChars(0, ' ');
tokenizer.CommentChar('#');
return(tokenizer);
}
/**
* /// Constructs an ExtendedStreamTokenizer from the given Reader. This ExtendedStreamTokenizer has no comment
* /// characters.
*
* /// @param reader the source of the data
* /// @param eolIsSignificant true if eol is significant
*/
public ExtendedStreamTokenizer(StreamReader reader, Boolean eolIsSignificant)
{
_reader = reader;
if (reader != null)
{
_tokenizer = new StreamTokenizer(reader);
_tokenizer.ResetSyntax();
_tokenizer.WhitespaceChars(0, 32);
_tokenizer.WordChars(33, 255);
_tokenizer.EolIsSignificant(eolIsSignificant);
}
else
{
_atEof = true;
}
_putbackList = new List <String>();
}
