public void TestTrailingWhiteSpace()
{
Queue queue = new Queue();
queue.Enqueue(new SimpleToken(TEXT, "package"));
queue.Enqueue(new SimpleToken(WS, " "));
queue.Enqueue(new SimpleToken(TEXT, "Empty"));
queue.Enqueue(new SimpleToken(WS, "\n\n\n")); // 1
queue.Enqueue(new Token(Token.EOF_TYPE)); // 2
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type); // 1)
AssertEquals(EOS, stream.nextToken().Type); // 2)
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}
void AssertTokenSequence(Token[] tokens, params int[] expectedSequence)
{
Queue queue = new Queue();
foreach (Token token in tokens)
{
queue.Enqueue(token);
}
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
int index = 0;
foreach (int expected in expectedSequence)
{
Assert.AreEqual(expected, stream.nextToken().Type, "sequence item: " + (index++));
}
}
public void TestClass()
{
Queue tokens = new Queue();
tokens.Enqueue(new SimpleToken(TEXT, "class"));
tokens.Enqueue(new SimpleToken(WS, " \t"));
tokens.Enqueue(new SimpleToken(TEXT, "foo:"));
tokens.Enqueue(new SimpleToken(WS, "\n\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "def foo():"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "pass"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\n\n\t")); // eos, d
tokens.Enqueue(new SimpleToken(TEXT, "def bar():"));
tokens.Enqueue(new SimpleToken(WS, "\n\t\t")); // i
tokens.Enqueue(new SimpleToken(TEXT, "pass"));
tokens.Enqueue(new Token(Token.EOF_TYPE)); // eos, d, d
TokenStream stream = new IndentTokenStreamFilter(new FakeStream(tokens), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}
public void TestMultipleDedent()
{
Queue queue = new Queue();
queue.Enqueue(new SimpleToken(TEXT, "class Math:"));
queue.Enqueue(new SimpleToken(WS, "\n\t"));
queue.Enqueue(new SimpleToken(TEXT, "def foo:"));
queue.Enqueue(new SimpleToken(WS, "\n\t\t"));
queue.Enqueue(new SimpleToken(TEXT, "pass"));
queue.Enqueue(new SimpleToken(WS, "\n"));
queue.Enqueue(new SimpleToken(TEXT, "print(3)"));
queue.Enqueue(new Token(Token.EOF_TYPE));
IndentTokenStreamFilter stream = new IndentTokenStreamFilter(new FakeStream(queue), WS, INDENT, DEDENT, EOS);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(INDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(DEDENT, stream.nextToken().Type);
AssertEquals(TEXT, stream.nextToken().Type);
AssertEquals(EOS, stream.nextToken().Type);
AssertEquals(Token.EOF_TYPE, stream.nextToken().Type);
}
