private void GenerateIndents(int amount)
{
for (int i = 0; i < amount; i++)
{
indentCount++;
output.Enqueue(new Token {
TokenType = TokenType.INDENT
});
}
index += amount * IndentLength;
}
public PQueue <Token> Tokenize(string input)
{
output = new PQueue <Token>();
TempPreprocess(ref input);
index = 0;
this.input = input;
Token nextToken;
while ((nextToken = FetchNextToken()) != null)
{
if (nextToken.TokenType != TokenType.WHITESPACE)
{
output.Enqueue(nextToken);
}
// If this is a new line, calculate the indentation level:
if (nextToken.TokenType == TokenType.NEWLINE)
{
GenerateIndentDedentTokens();
}
}
return(output);
}
