public void LiteralStringBalanced3() { Tokenizer t = new Tokenizer(StringToStream("(a(bc)d)")); TokenStringLiteral s = t.GetToken() as TokenStringLiteral; Assert.NotNull(s); Assert.True(s.Raw == "a(bc)d"); Assert.True(s.Resolved == "a(bc)d"); Assert.True(t.GetToken() is TokenEmpty); }
public void LiteralStringContinuation() { Tokenizer t = new Tokenizer(StringToStream(" (a\\\nb\\\nc) ")); TokenStringLiteral s = t.GetToken() as TokenStringLiteral; Assert.NotNull(s); Assert.True(s.Raw == "abc"); Assert.True(s.Resolved == "abc"); Assert.True(t.GetToken() is TokenEmpty); }
public void LiteralStringEscapedOctal7() { Tokenizer t = new Tokenizer(StringToStream("(abc\\0def)")); TokenStringLiteral s = t.GetToken() as TokenStringLiteral; Assert.NotNull(s); Assert.True(s.Raw == "abc\\0def"); Assert.True(s.Resolved == "abc\0def"); Assert.True(t.GetToken() is TokenEmpty); }
public void UTF16LittleEndianD() { Tokenizer t = new Tokenizer(BytesToStream(new byte[] { 0x28, // ( 0xFF, 0xFE, // UTF16 Byte Order Mark 0x52, 0xD8, 0x62, 0xDF, // 𤭢 0x29 // ) })); TokenStringLiteral s = t.GetToken() as TokenStringLiteral; Assert.NotNull(s); Assert.True(s.Resolved == "𤭢"); Assert.True(t.GetToken() is TokenEmpty); }
public void UTF16BigEndianC() { Tokenizer t = new Tokenizer(BytesToStream(new byte[] { 0x28, // ( 0xFE, 0xFF, // UTF16 Byte Order Mark 0xD8, 0x01, 0xDC, 0x37, // 𐐷 0x29 // ) })); TokenStringLiteral s = t.GetToken() as TokenStringLiteral; Assert.NotNull(s); Assert.True(s.Resolved == "𐐷"); Assert.True(t.GetToken() is TokenEmpty); }
private List <object> parseInclude(PreprocessorDirective dir) { if (dir.Operands.Count != 1 && !(dir.Operands[0] is TokenStringLiteral)) { return(new List <object>()); } else { TokenStringLiteral sl = dir.Operands[0] as TokenStringLiteral; string source = File.ReadAllText(sl.Value); Lexer lex = new Lexer(source); lex.Scan(); Parser parser = new Parser(); parser.ProcessTokens(lex.TokenList); return(parser.instructions); } }