public Token GetNextToken()
{
var location = Location;
var symbol = GoldParser.ReadToken();
Token token = null;
var tokenType = (TokenType)symbol.Index;
// Else process the symbol as it should
switch (symbol.SymbolType)
{
case SymbolType.WhiteSpace:
token = new Token(languageData.FindTerminalByType(tokenType), location, GoldParser.TokenLength, source, null);
break;
case SymbolType.CommentLine:
case SymbolType.CommentStart:
int length = GoldParser.CommentTextLength(location.Position) - location.Position;
token = new Token(languageData.FindTerminalByType(tokenType), location, length, source, null);
break;
case SymbolType.Error:
token = new Token(languageData.Grammar.SyntaxError, location, GoldParser.TokenLength, source, "Unexpected token");
break;
case SymbolType.End:
token = new Token(languageData.Grammar.Eof, location, string.Empty, languageData.Grammar.Eof.Name);
break;
default:
// Skip preprocessor line
// Update line number according to
if (symbol.Index == (int)TokenType.Preprocessor)
{
int tempPreviousLine = GoldParser.LineNumber;
bool isEndOfLine = false;
bool preprocessorDecoded = false;
while (!isEndOfLine)
{
symbol = GoldParser.ReadToken();
tokenType = (TokenType)symbol.Index;
switch ((TokenType)symbol.Index)
{
case TokenType.Eof:
case TokenType.NewLine:
isEndOfLine = true;
break;
case TokenType.Identifier:
if (!preprocessorDecoded)
{
preprocessorDecoded = GoldParser.TokenText != "line";
}
break;
case TokenType.StringLiteral:
if (preprocessorDecoded)
{
sourceFileName = GoldParser.TokenText.Trim('"').Replace(@"\\", @"\");
}
break;
case TokenType.WS:
case TokenType.Whitespace:
break;
case TokenType.StartWithNoZeroDecimalIntegerLiteral:
if (!preprocessorDecoded)
{
previousLine = tempPreviousLine;
newLine = int.Parse(GoldParser.TokenText) - 1;
preprocessorDecoded = true;
}
break;
default:
preprocessorDecoded = true;
break;
}
}
location = Location;
}
Terminal terminal = null;
if (tokenType == TokenType.Identifier)
{
terminal = languageData.FindTerminalByName(GoldParser.TokenText);
}
if (terminal == null)
{
terminal = languageData.FindTerminalByType((TokenType)symbol.Index);
}
if (terminal == null)
{
token = new Token(languageData.Grammar.SyntaxError, location, GoldParser.TokenText, string.Format("Unable to find terminal for token text [{0}]", GoldParser.TokenText));
}
else
{
if (terminal is DynamicKeyTerm)
{
((DynamicKeyTerm)terminal).Match(this, out token);
}
else
{
token = new Token(terminal, location, GoldParser.TokenLength, source, null);
}
}
break;
}
return(token);
}
