private void GenerateIndents(int amount) { for (int i = 0; i < amount; i++) { indentCount++; output.Enqueue(new Token { TokenType = TokenType.INDENT }); } index += amount * IndentLength; }
public PQueue <Token> Tokenize(string input) { output = new PQueue <Token>(); TempPreprocess(ref input); index = 0; this.input = input; Token nextToken; while ((nextToken = FetchNextToken()) != null) { if (nextToken.TokenType != TokenType.WHITESPACE) { output.Enqueue(nextToken); } // If this is a new line, calculate the indentation level: if (nextToken.TokenType == TokenType.NEWLINE) { GenerateIndentDedentTokens(); } } return(output); }