private Ast.Identifier ParseIdentifier(Token token, bool allowIndexed = false)
{
var identifierName = new System.Text.StringBuilder(token.Value);
// ------------------------------------------------------------
// Parse complex identifier XXXX::AAAA::BBBBB
// ------------------------------------------------------------
BeginPreviewToken();
var nextToken = NextToken();
EndPreviewToken();
// Dot are not supported by compiler XXXX.AAAA.BBBB
// if (nextToken.Type == TokenType.DoubleColon || nextToken.Type == TokenType.Dot)
if (nextToken.Type == TokenType.DoubleColon)
{
var identifierSeparatorType = nextToken.Type;
var endChar = nextToken.Type == TokenType.DoubleColon ? "::" : ".";
nextToken = NextToken();
identifierName.Append(nextToken.Value);
bool continueParsing = true;
do
{
BeginPreviewToken();
nextToken = NextToken();
EndPreviewToken();
switch (nextToken.Type)
{
case TokenType.Identifier:
identifierName.Append(nextToken.Value);
break;
case TokenType.DoubleColon:
case TokenType.Dot:
if (identifierName[identifierName.Length - 1] == endChar[0])
{
Logger.Error("Unexpected token [{0}]. Expecting tokens [identifier]", nextToken.Span, endChar);
}
if (identifierSeparatorType != nextToken.Type)
{
Logger.Error("Unexpected token [{0}]. Expecting tokens [{1}]", nextToken.Span, nextToken.Value, endChar);
}
identifierName.Append(nextToken.Value);
break;
default:
continueParsing = false;
break;
}
// If can go to the next token
if (continueParsing)
{
nextToken = NextToken();
}
} while (continueParsing);
if (identifierName[identifierName.Length - 1] == endChar[0])
{
Logger.Error("Unexpected token [{0}]. Expecting tokens [identifier]", nextToken.Span, endChar);
}
}
// ------------------------------------------------------------
// Parse optional indexer a[xxx]
// ------------------------------------------------------------
if (allowIndexed)
{
BeginPreviewToken();
var arrayToken = NextToken();
EndPreviewToken();
if (arrayToken.Type == TokenType.LeftBracket)
{
// Skip left bracket [
NextToken();
// Get first token from inside bracket
NextToken();
// Parse value inside bracket
var expression = ParseExpression() as Ast.LiteralExpression;
if (expression == null || !(expression.Value.Value is int))
{
Logger.Error("Unsupported expression for indexed variable. Only support for integer Literal", currentToken.Span);
return null;
}
ExpectNext(TokenType.RightBracket);
return new Ast.IndexedIdentifier(identifierName.ToString(), (int)expression.Value.Value) { Span = token.Span };
}
}
return new Ast.Identifier(identifierName.ToString()) { Span = token.Span };
}
private Token InternalNextToken()
{
// TODO: token preview is not safe with NewLine count
if (previewTokens.Count > 0)
{
currentToken = previewTokens[0];
previewTokens.RemoveAt(0);
if (isPreviewToken)
currentPreviewTokens.Add(currentToken);
return currentToken;
}
while (tokenEnumerator.MoveNext())
{
currentToken = tokenEnumerator.Current;
if (currentToken.Type == TokenType.Newline)
{
currentLine++;
currentLineAbsolutePos = currentToken.Span.Index + currentToken.Span.Length;
}
else
{
currentToken.Span.Line = currentLine;
currentToken.Span.Column = currentToken.Span.Index - currentLineAbsolutePos + 1;
currentToken.Span.FilePath = currentFile;
HandleBrackets(currentToken);
if (isPreviewToken)
currentPreviewTokens.Add(currentToken);
return currentToken;
}
}
currentToken = EndOfFile;
return currentToken;
}
private void BeginPreviewToken()
{
isPreviewToken = true;
savedPreviewToken = currentToken;
}
private void EndPreviewToken()
{
isPreviewToken = false;
previewTokens = currentPreviewTokens;
currentPreviewTokens = new List<Token>();
currentToken = savedPreviewToken;
}
private bool HandleBrackets(Token token)
{
bool isBracketOk = true;
switch (token.Type)
{
case TokenType.LeftParent:
lastParentToken = token;
parentCount++;
break;
case TokenType.RightParent:
lastParentToken = token;
parentCount--;
if (parentCount < 0)
{
parentCount = 0;
Logger.Error("No matching parenthesis '(' for this closing token.", token.Span);
isBracketOk = false;
}
break;
case TokenType.LeftBracket:
lastBracketToken = token;
bracketCount++;
break;
case TokenType.RightBracket:
lastBracketToken = token;
bracketCount--;
if (bracketCount < 0)
{
bracketCount = 0;
Logger.Error("No matching bracket '[' for this closing token.", token.Span);
isBracketOk = false;
}
break;
case TokenType.LeftCurlyBrace:
lastCurlyBraceToken = token;
curlyBraceCount++;
break;
case TokenType.RightCurlyBrace:
lastCurlyBraceToken = token;
curlyBraceCount--;
if (curlyBraceCount < 0)
{
curlyBraceCount = 0;
Logger.Error("No matching curly brace '{{' for this closing token.", token.Span);
isBracketOk = false;
}
break;
}
return isBracketOk;
}
