public void TestQuote() { const string testStr = "token1 token2 \"The test string\" token4"; _outWriter("Parsing String: " + testStr); StreamTokenizer st = new StreamTokenizer(new MemoryStream( Encoding.UTF8.GetBytes(testStr))); bool foundToken = false; String matchStr = null; while (st.NextToken() != StreamTokenizer.TT_EOF) { switch (st.ttype) { case '\"': foundToken = true; matchStr = st.ToString(); _outWriter("Found token " + matchStr); break; default: _outWriter("Found token " + st); break; } } if (!foundToken) { throw new Exception("Test failed to recognize Quote type"); } if (!matchStr.Equals("Token[The test string], line 1")) { throw new Exception("Test failed parse quoted string"); } }
public void TestNullConstruct() { Assert.Throws <ArgumentNullException>(() => { StreamTokenizer st = new StreamTokenizer(null); }); }
public string ParseExpression(Stream stm) { var buff = new StringBuilder(); var st = new StreamTokenizer(stm); return(buff.ToString()); }
public void TestComments() { var res = "ACMESharp.Util.StreamTokenizer.input.txt"; var asm = typeof(StreamTokenizerTests).GetTypeInfo().Assembly; int slashIsCommentStart = 1; int slashSlashComment = 2; int slashStarComment = 4; for (int i = 0; i < 8; i++) { using (var stm = asm.GetManifestResourceStream(res)) { var st = new StreamTokenizer(stm); /* decide the state of this run */ bool slashCommentFlag = ((i & slashIsCommentStart) != 0); bool slashSlashCommentFlag = ((i & slashSlashComment) != 0); bool slashStarCommentFlag = ((i & slashStarComment) != 0); /* set the initial state of the tokenizer */ if (!slashCommentFlag) { st.OrdinaryChars('/', '/'); } st.SlashSlashComments(slashSlashCommentFlag); st.SlashStarComments(slashStarCommentFlag); /* now go throgh the input file */ while (st.NextToken() != StreamTokenizer.TT_EOF) { String token = st.sval; if (token == null) { continue; } else { if ((token.CompareTo("Error1") == 0) && slashStarCommentFlag) { throw new Exception("Failed to pass one line C comments!"); } if ((token.CompareTo("Error2") == 0) && slashStarCommentFlag) { throw new Exception("Failed to pass multi line C comments!"); } if ((token.CompareTo("Error3") == 0) && slashSlashCommentFlag) { throw new Exception("Failed to pass C++ comments!"); } if ((token.CompareTo("Error4") == 0) && slashCommentFlag) { throw new Exception("Failed to pass / comments!"); } } } } } }
public void TestReset() { var stm = new MemoryStream(Encoding.UTF8.GetBytes("[ #")); var scan = new StreamTokenizer(stm); scan.NextToken(); scan.NextToken(); stm.Seek(0, SeekOrigin.Begin); int token = scan.NextToken(); Assert.Equal('[', token); }
public ExpressionParser(Stream stream, IDictionary <string, object> map, Action <string> outWriter = null) { _stream = stream; _map = map; _tok = new StreamTokenizer(_stream); //_tok.ResetSyntax(); if (outWriter == null) { outWriter = Console.WriteLine; } _outWriter = outWriter; _buff = new StringBuilder(); }
public void TestReadAhead() { foreach (var s in new[] { "foo\nx", "foo\r\nx" }) { _outWriter("Testing:"); _outWriter(s); using (var lis = new LimitedInputStream(s, 4)) { var st = new StreamTokenizer(lis); st.EolIsSignificant(true); int tt = st.NextToken(); Assert.Equal(StreamTokenizer.TT_WORD, tt); Assert.Equal("foo", st.sval); tt = st.NextToken(); Assert.Equal(StreamTokenizer.TT_EOL, tt); } } }