Esempio n. 1
0
        public void Test_ConstructorLSystem_IO_InputStream()
        {
            st = new StreamTokenizer(new MemoryStream(
                                         Encoding.UTF8.GetBytes("/comments\n d 8 'h'")));


            assertEquals("the next token returned should be the letter d",
                         StreamTokenizer.TokenType_Word, st.NextToken());

            assertEquals("the next token returned should be the letter d",
                         "d", st.StringValue);


            assertEquals("the next token returned should be the digit 8",
                         StreamTokenizer.TokenType_Number, st.NextToken());

            assertEquals("the next token returned should be the digit 8",
                         8.0, st.NumberValue);


            assertEquals("the next token returned should be the quote character",
                         39, st.NextToken());

            assertEquals("the next token returned should be the quote character",
                         "h", st.StringValue);
        }
Esempio n. 2
0
        public void Test_slashStarComments_withSTClosed()
        {
            TextReader reader = new StringReader("t /* t */ t");

            StreamTokenizer st = new StreamTokenizer(reader);

            st.SlashStarComments = (false);

            assertEquals(StreamTokenizer.TokenType_Word, st.NextToken());
            assertEquals(StreamTokenizer.TokenType_EndOfStream, st.NextToken());
        }
Esempio n. 3
0
        public void Test_slashSlashComments_withSSOpen_NoComment()
        {
            TextReader reader = new StringReader("// t");

            StreamTokenizer st = new StreamTokenizer(reader);

            st.SlashSlashComments = (true);
            st.OrdinaryChar('/');

            assertEquals(StreamTokenizer.TokenType_EndOfStream, st.NextToken());
        }
Esempio n. 4
0
        public void Test_slashSlashComments_withSSOpen()
        {
            TextReader reader = new StringReader("t // t t t");

            StreamTokenizer st = new StreamTokenizer(reader);

            st.SlashSlashComments = (true);

            assertEquals(StreamTokenizer.TokenType_Word, st.NextToken());
            assertEquals(StreamTokenizer.TokenType_EndOfStream, st.NextToken());
        }
Esempio n. 5
0
        public void Test_slashSlashComments_withSSClosed()
        {
            TextReader reader = new StringReader("// t");

            StreamTokenizer st = new StreamTokenizer(reader);

            st.SlashSlashComments = (false);
            st.OrdinaryChar('/');

            assertEquals('/', st.NextToken());
            assertEquals('/', st.NextToken());
            assertEquals(StreamTokenizer.TokenType_Word, st.NextToken());
        }
Esempio n. 6
0
        public void Test_toString()
        {
            setTest("ABC Hello World");
            st.NextToken();
            assertTrue("toString failed." + st.ToString(),
                       st.ToString().Equals(
                           "Token[ABC], line 1", StringComparison.Ordinal));

            // Regression test for HARMONY-4070
            byte[]          data      = new byte[] { (byte)'-' };
            StreamTokenizer tokenizer = new StreamTokenizer(
                new MemoryStream(data));

            tokenizer.NextToken();
            String result = tokenizer.ToString();

            assertEquals("Token['-'], line 1", result);
        }
Esempio n. 7
0
 private void setTest(string s)
 {
     testString = s;
     r          = new StringReader(testString);
     st         = new StreamTokenizer(r);
 }
Esempio n. 8
0
        public void Test_nextToken()
        {
            // SM.
            // J2N NOTE: The original test had \257 (which is octal)
            // that is not supported in a .NET string, so we convert to decimal 175 here.
            // This also changes the semantics of the test, because for whatever
            // reason in Java it was expecting the octal number to register as a TokenType_Word.
            // So, we changed to expect a TokenType_Number as a result of the above change.
            // Also, we don't need to escape single quotes in .NET.
            setTest("\r\n/* fje fje 43.4 f \r\n f g */  456.459 \r\n"
                    + "Hello  / 	\r\n \r\n \n \r 175 Hi 'Hello World'");
            st.OrdinaryChar('/');
            st.SlashStarComments = true;
            st.NextToken();
            assertTrue("Wrong Token type1: " + (char)st.TokenType,
                       st.TokenType == StreamTokenizer.TokenType_Number);
            st.NextToken();
            assertTrue("Wrong Token type2: " + st.TokenType,
                       st.TokenType == StreamTokenizer.TokenType_Word);
            st.NextToken();
            assertTrue("Wrong Token type3: " + st.TokenType, st.TokenType == '/');
            st.NextToken();
            assertTrue("Wrong Token type4: " + st.TokenType,
                       st.TokenType == StreamTokenizer.TokenType_Number);
            st.NextToken();
            assertTrue("Wrong Token type5: " + st.TokenType,
                       st.TokenType == StreamTokenizer.TokenType_Word);
            st.NextToken();
            assertTrue("Wrong Token type6: " + st.TokenType, st.TokenType == '\'');
            assertTrue("Wrong Token type7: " + st.TokenType, st.StringValue
                       .Equals("Hello World", StringComparison.Ordinal));
            st.NextToken();
            assertTrue("Wrong Token type8: " + st.TokenType, st.TokenType == -1);

            using (var pin = new MemoryStream(Encoding.UTF8.GetBytes("hello\n\r\r")))
            {
                StreamTokenizer s = new StreamTokenizer(pin);
                s.EndOfLineIsSignificant = (true);

                assertTrue("Wrong token 1,1",
                           s.NextToken() == StreamTokenizer.TokenType_Word &&
                           s.StringValue.Equals("hello", StringComparison.Ordinal));

                assertTrue("Wrong token 1,2", s.NextToken() == '\n');

                assertTrue("Wrong token 1,3", s.NextToken() == '\n');

                assertTrue("Wrong token 1,4", s.NextToken() == '\n');


                assertTrue("Wrong token 1,5",
                           s.NextToken() == StreamTokenizer.TokenType_EndOfStream);
            }
            StreamTokenizer tokenizer = new StreamTokenizer(
                new StringReader("\n \r\n#"));

            tokenizer.OrdinaryChar('\n'); // make \n ordinary
            tokenizer.EndOfLineIsSignificant = (true);

            assertTrue("Wrong token 2,1", tokenizer.NextToken() == '\n');

            assertTrue("Wrong token 2,2", tokenizer.NextToken() == '\n');

            assertEquals("Wrong token 2,3", '#', tokenizer.NextToken());
        }