public Token GetNextToken() { var location = Location; var symbol = GoldParser.ReadToken(); Token token = null; var tokenType = (TokenType)symbol.Index; // Else process the symbol as it should switch (symbol.SymbolType) { case SymbolType.WhiteSpace: token = new Token(languageData.FindTerminalByType(tokenType), location, GoldParser.TokenLength, source, null); break; case SymbolType.CommentLine: case SymbolType.CommentStart: int length = GoldParser.CommentTextLength(location.Position) - location.Position; token = new Token(languageData.FindTerminalByType(tokenType), location, length, source, null); break; case SymbolType.Error: token = new Token(languageData.Grammar.SyntaxError, location, GoldParser.TokenLength, source, "Unexpected token"); break; case SymbolType.End: token = new Token(languageData.Grammar.Eof, location, string.Empty, languageData.Grammar.Eof.Name); break; default: // Skip preprocessor line // Update line number according to if (symbol.Index == (int)TokenType.Preprocessor) { int tempPreviousLine = GoldParser.LineNumber; bool isEndOfLine = false; bool preprocessorDecoded = false; while (!isEndOfLine) { symbol = GoldParser.ReadToken(); tokenType = (TokenType)symbol.Index; switch ((TokenType)symbol.Index) { case TokenType.Eof: case TokenType.NewLine: isEndOfLine = true; break; case TokenType.Identifier: if (!preprocessorDecoded) { preprocessorDecoded = GoldParser.TokenText != "line"; } break; case TokenType.StringLiteral: if (preprocessorDecoded) { sourceFileName = GoldParser.TokenText.Trim('"').Replace(@"\\", @"\"); } break; case TokenType.WS: case TokenType.Whitespace: break; case TokenType.StartWithNoZeroDecimalIntegerLiteral: if (!preprocessorDecoded) { previousLine = tempPreviousLine; newLine = int.Parse(GoldParser.TokenText) - 1; preprocessorDecoded = true; } break; default: preprocessorDecoded = true; break; } } location = Location; } Terminal terminal = null; if (tokenType == TokenType.Identifier) { terminal = languageData.FindTerminalByName(GoldParser.TokenText); } if (terminal == null) { terminal = languageData.FindTerminalByType((TokenType)symbol.Index); } if (terminal == null) { token = new Token(languageData.Grammar.SyntaxError, location, GoldParser.TokenText, string.Format("Unable to find terminal for token text [{0}]", GoldParser.TokenText)); } else { if (terminal is DynamicKeyTerm) { ((DynamicKeyTerm)terminal).Match(this, out token); } else { token = new Token(terminal, location, GoldParser.TokenLength, source, null); } } break; } return(token); }