public void LiteralStringBalanced3()
{
Tokenizer t = new Tokenizer(StringToStream("(a(bc)d)"));
TokenStringLiteral s = t.GetToken() as TokenStringLiteral;
Assert.NotNull(s);
Assert.True(s.Raw == "a(bc)d");
Assert.True(s.Resolved == "a(bc)d");
Assert.True(t.GetToken() is TokenEmpty);
}
public void LiteralStringContinuation()
{
Tokenizer t = new Tokenizer(StringToStream(" (a\\\nb\\\nc) "));
TokenStringLiteral s = t.GetToken() as TokenStringLiteral;
Assert.NotNull(s);
Assert.True(s.Raw == "abc");
Assert.True(s.Resolved == "abc");
Assert.True(t.GetToken() is TokenEmpty);
}
public void LiteralStringEscapedOctal7()
{
Tokenizer t = new Tokenizer(StringToStream("(abc\\0def)"));
TokenStringLiteral s = t.GetToken() as TokenStringLiteral;
Assert.NotNull(s);
Assert.True(s.Raw == "abc\\0def");
Assert.True(s.Resolved == "abc\0def");
Assert.True(t.GetToken() is TokenEmpty);
}
public void UTF16LittleEndianD()
{
Tokenizer t = new Tokenizer(BytesToStream(new byte[] {
0x28, // (
0xFF, 0xFE, // UTF16 Byte Order Mark
0x52, 0xD8, 0x62, 0xDF, // 𤭢
0x29 // )
}));
TokenStringLiteral s = t.GetToken() as TokenStringLiteral;
Assert.NotNull(s);
Assert.True(s.Resolved == "𤭢");
Assert.True(t.GetToken() is TokenEmpty);
}
public void UTF16BigEndianC()
{
Tokenizer t = new Tokenizer(BytesToStream(new byte[] {
0x28, // (
0xFE, 0xFF, // UTF16 Byte Order Mark
0xD8, 0x01, 0xDC, 0x37, // 𐐷
0x29 // )
}));
TokenStringLiteral s = t.GetToken() as TokenStringLiteral;
Assert.NotNull(s);
Assert.True(s.Resolved == "𐐷");
Assert.True(t.GetToken() is TokenEmpty);
}
private List <object> parseInclude(PreprocessorDirective dir)
{
if (dir.Operands.Count != 1 && !(dir.Operands[0] is TokenStringLiteral))
{
return(new List <object>());
}
else
{
TokenStringLiteral sl = dir.Operands[0] as TokenStringLiteral;
string source = File.ReadAllText(sl.Value);
Lexer lex = new Lexer(source);
lex.Scan();
Parser parser = new Parser();
parser.ProcessTokens(lex.TokenList);
return(parser.instructions);
}
}
