public PQueue <Token> Tokenize(string input)
{
var queue = new PQueue <Token>();
foreach (var token in LexerOutput)
{
queue.Enqueue(token);
}
return(queue);
}
public PQueue <Token> Tokenize(string input)
{
var output = new PQueue <Token>();
index = 0;
this.input = input;
Token nextToken;
while ((nextToken = FetchNextToken()) != null)
{
if (nextToken.TokenType != TokenType.WHITESPACE)
{
output.Enqueue(nextToken);
}
}
return(output);
}