public void TestClass()
{
Queue tokens = new Queue();
tokens.Enqueue(new SimpleToken(TEXT, "class"));
tokens.Enqueue(new SimpleToken(WS, " \t"));
tokens.Enqueue(new SimpleToken(TEXT, "foo:"));
tokens.Enqueue(new SimpleToken(WS, "\n\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "def foo():"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "pass"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\n\n\t")); // eos, d
tokens.Enqueue(new SimpleToken(TEXT, "def bar():"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "pass"));
tokens.Enqueue(new Token(Token.EOF_TYPE)); // eos, d, d
TokenStream stream = new IndentTokenStreamFilter(new FakeStream(tokens), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}
public void TestMultipleDedent()
{
Queue queue = new Queue();
queue.Enqueue(new SimpleToken(TEXT, "class Math:"));
queue.Enqueue(new SimpleToken(WS, "\n\t"));
queue.Enqueue(new SimpleToken(TEXT, "def foo:"));
queue.Enqueue(new SimpleToken(WS, "\n\t\t"));
queue.Enqueue(new SimpleToken(TEXT, "pass"));
queue.Enqueue(new SimpleToken(WS, "\n"));
queue.Enqueue(new SimpleToken(TEXT, "print(3)"));
queue.Enqueue(new Token(Token.EOF_TYPE));
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}
public static Module ParseModule(string readerName, TextReader reader, ParserErrorHandler errorHandler)
{
antlr.TokenStreamSelector selector = new antlr.TokenStreamSelector();
BooLexer lexer = new BooLexer(reader);
lexer.setTabSize(TabSize);
lexer.setFilename(readerName);
lexer.setTokenObjectClass(TokenObjectClass);
lexer.Initialize(selector, TabSize, TokenObjectClass);
IndentTokenStreamFilter filter = new IndentTokenStreamFilter(lexer, WS, INDENT, DEDENT, EOS);
selector.select(filter);
BooParser parser = new BooParser(selector);
parser.setFilename(readerName);
parser.Error += errorHandler;
Module module = parser.start();
module.Name = CreateModuleName(readerName);
return(module);
}
public static antlr.TokenStream CreateBooLexer(int tabSize, string readerName, TextReader reader)
{
antlr.TokenStreamSelector selector = new antlr.TokenStreamSelector();
BooLexer lexer = new BooLexer(reader);
lexer.setFilename(readerName);
lexer.Initialize(selector, tabSize, BooToken.TokenCreator);
IndentTokenStreamFilter filter = new IndentTokenStreamFilter(lexer, WS, INDENT, DEDENT, EOL);
selector.select(filter);
return(selector);
}
void AssertTokenSequence(Token[] tokens, params int[] expectedSequence)
{
Queue queue = new Queue();
foreach (Token token in tokens)
{
queue.Enqueue(token);
}
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
int index = 0;
foreach (int expected in expectedSequence)
{
Assert.AreEqual(expected, stream.nextToken().Type, "sequence item: " + (index++));
}
}
public void TestTrailingWhiteSpace()
{
Queue queue = new Queue();
queue.Enqueue(new SimpleToken(TEXT, "package"));
queue.Enqueue(new SimpleToken(WS, " "));
queue.Enqueue(new SimpleToken(TEXT, "Empty"));
queue.Enqueue(new SimpleToken(WS, "\n\n\n")); // 1
queue.Enqueue(new Token(Token.EOF_TYPE)); // 2
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type); // 1)
AssertEquals(EOS, stream.nextToken().Type); // 2)
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}