public void Test_commentCharI()
{
setTest("*comment \n / 8 'h' ");
st.OrdinaryChar('/');
st.CommentChar('*');
assertEquals("nextToken() did not return the character / skiping the comments starting with *",
47, st.NextToken());
assertTrue("the next token returned should be the digit 8", st
.NextToken() == StreamTokenizer.TT_NUMBER &&
st.NumberValue == 8.0);
assertTrue("the next token returned should be the quote character",
st.NextToken() == 39 && st.StringValue.Equals("h", StringComparison.Ordinal));
}
public void Test_slashSlashComments_withSSOpen_NoComment()
{
TextReader reader = new StringReader("// t");
StreamTokenizer st = new StreamTokenizer(reader);
st.SlashSlashComments = (true);
st.OrdinaryChar('/');
assertEquals(StreamTokenizer.TT_EOF, st.NextToken());
}
public void Test_slashSlashComments_withSSClosed()
{
TextReader reader = new StringReader("// t");
StreamTokenizer st = new StreamTokenizer(reader);
st.SlashSlashComments = (false);
st.OrdinaryChar('/');
assertEquals('/', st.NextToken());
assertEquals('/', st.NextToken());
assertEquals(StreamTokenizer.TT_WORD, st.NextToken());
}
public void Test_nextToken()
{
// SM.
// LUCENENET NOTE: The original test had \257 (which is octal)
// that is not supported in a .NET string, so we convert to decimal 175 here.
// This also changes the semantics of the test, because for whatever
// reason in Java it was expecting the octal number to register as a TT_WORD.
// So, we changed to expect a TT_NUMBER as a result of the above change.
// Also, we don't need to escape single quotes in .NET.
setTest("\r\n/* fje fje 43.4 f \r\n f g */ 456.459 \r\n"
+ "Hello / \r\n \r\n \n \r 175 Hi 'Hello World'");
st.OrdinaryChar('/');
st.SlashStarComments = true;
st.NextToken();
assertTrue("Wrong Token type1: " + (char)st.TokenType,
st.TokenType == StreamTokenizer.TT_NUMBER);
st.NextToken();
assertTrue("Wrong Token type2: " + st.TokenType,
st.TokenType == StreamTokenizer.TT_WORD);
st.NextToken();
assertTrue("Wrong Token type3: " + st.TokenType, st.TokenType == '/');
st.NextToken();
assertTrue("Wrong Token type4: " + st.TokenType,
st.TokenType == StreamTokenizer.TT_NUMBER);
st.NextToken();
assertTrue("Wrong Token type5: " + st.TokenType,
st.TokenType == StreamTokenizer.TT_WORD);
st.NextToken();
assertTrue("Wrong Token type6: " + st.TokenType, st.TokenType == '\'');
assertTrue("Wrong Token type7: " + st.TokenType, st.StringValue
.Equals("Hello World", StringComparison.Ordinal));
st.NextToken();
assertTrue("Wrong Token type8: " + st.TokenType, st.TokenType == -1);
using (var pin = new MemoryStream(Encoding.UTF8.GetBytes("hello\n\r\r")))
{
#pragma warning disable 612, 618
StreamTokenizer s = new StreamTokenizer(pin);
#pragma warning restore 612, 618
s.IsEOLSignificant = (true);
assertTrue("Wrong token 1,1",
s.NextToken() == StreamTokenizer.TT_WORD &&
s.StringValue.Equals("hello", StringComparison.Ordinal));
assertTrue("Wrong token 1,2", s.NextToken() == '\n');
assertTrue("Wrong token 1,3", s.NextToken() == '\n');
assertTrue("Wrong token 1,4", s.NextToken() == '\n');
assertTrue("Wrong token 1,5",
s.NextToken() == StreamTokenizer.TT_EOF);
}
StreamTokenizer tokenizer = new StreamTokenizer(
new StringReader("\n \r\n#"));
tokenizer.OrdinaryChar('\n'); // make \n ordinary
tokenizer.IsEOLSignificant = (true);
assertTrue("Wrong token 2,1", tokenizer.NextToken() == '\n');
assertTrue("Wrong token 2,2", tokenizer.NextToken() == '\n');
assertEquals("Wrong token 2,3", '#', tokenizer.NextToken());
}