private Ast.Identifier ParseIdentifier(Token token, bool allowIndexed = false) { var identifierName = new System.Text.StringBuilder(token.Value); // ------------------------------------------------------------ // Parse complex identifier XXXX::AAAA::BBBBB // ------------------------------------------------------------ BeginPreviewToken(); var nextToken = NextToken(); EndPreviewToken(); // Dot are not supported by compiler XXXX.AAAA.BBBB // if (nextToken.Type == TokenType.DoubleColon || nextToken.Type == TokenType.Dot) if (nextToken.Type == TokenType.DoubleColon) { var identifierSeparatorType = nextToken.Type; var endChar = nextToken.Type == TokenType.DoubleColon ? "::" : "."; nextToken = NextToken(); identifierName.Append(nextToken.Value); bool continueParsing = true; do { BeginPreviewToken(); nextToken = NextToken(); EndPreviewToken(); switch (nextToken.Type) { case TokenType.Identifier: identifierName.Append(nextToken.Value); break; case TokenType.DoubleColon: case TokenType.Dot: if (identifierName[identifierName.Length - 1] == endChar[0]) { Logger.Error("Unexpected token [{0}]. Expecting tokens [identifier]", nextToken.Span, endChar); } if (identifierSeparatorType != nextToken.Type) { Logger.Error("Unexpected token [{0}]. Expecting tokens [{1}]", nextToken.Span, nextToken.Value, endChar); } identifierName.Append(nextToken.Value); break; default: continueParsing = false; break; } // If can go to the next token if (continueParsing) { nextToken = NextToken(); } } while (continueParsing); if (identifierName[identifierName.Length - 1] == endChar[0]) { Logger.Error("Unexpected token [{0}]. Expecting tokens [identifier]", nextToken.Span, endChar); } } // ------------------------------------------------------------ // Parse optional indexer a[xxx] // ------------------------------------------------------------ if (allowIndexed) { BeginPreviewToken(); var arrayToken = NextToken(); EndPreviewToken(); if (arrayToken.Type == TokenType.LeftBracket) { // Skip left bracket [ NextToken(); // Get first token from inside bracket NextToken(); // Parse value inside bracket var expression = ParseExpression() as Ast.LiteralExpression; if (expression == null || !(expression.Value.Value is int)) { Logger.Error("Unsupported expression for indexed variable. Only support for integer Literal", currentToken.Span); return null; } ExpectNext(TokenType.RightBracket); return new Ast.IndexedIdentifier(identifierName.ToString(), (int)expression.Value.Value) { Span = token.Span }; } } return new Ast.Identifier(identifierName.ToString()) { Span = token.Span }; }
private Token InternalNextToken() { // TODO: token preview is not safe with NewLine count if (previewTokens.Count > 0) { currentToken = previewTokens[0]; previewTokens.RemoveAt(0); if (isPreviewToken) currentPreviewTokens.Add(currentToken); return currentToken; } while (tokenEnumerator.MoveNext()) { currentToken = tokenEnumerator.Current; if (currentToken.Type == TokenType.Newline) { currentLine++; currentLineAbsolutePos = currentToken.Span.Index + currentToken.Span.Length; } else { currentToken.Span.Line = currentLine; currentToken.Span.Column = currentToken.Span.Index - currentLineAbsolutePos + 1; currentToken.Span.FilePath = currentFile; HandleBrackets(currentToken); if (isPreviewToken) currentPreviewTokens.Add(currentToken); return currentToken; } } currentToken = EndOfFile; return currentToken; }
private void BeginPreviewToken() { isPreviewToken = true; savedPreviewToken = currentToken; }
private void EndPreviewToken() { isPreviewToken = false; previewTokens = currentPreviewTokens; currentPreviewTokens = new List<Token>(); currentToken = savedPreviewToken; }
private bool HandleBrackets(Token token) { bool isBracketOk = true; switch (token.Type) { case TokenType.LeftParent: lastParentToken = token; parentCount++; break; case TokenType.RightParent: lastParentToken = token; parentCount--; if (parentCount < 0) { parentCount = 0; Logger.Error("No matching parenthesis '(' for this closing token.", token.Span); isBracketOk = false; } break; case TokenType.LeftBracket: lastBracketToken = token; bracketCount++; break; case TokenType.RightBracket: lastBracketToken = token; bracketCount--; if (bracketCount < 0) { bracketCount = 0; Logger.Error("No matching bracket '[' for this closing token.", token.Span); isBracketOk = false; } break; case TokenType.LeftCurlyBrace: lastCurlyBraceToken = token; curlyBraceCount++; break; case TokenType.RightCurlyBrace: lastCurlyBraceToken = token; curlyBraceCount--; if (curlyBraceCount < 0) { curlyBraceCount = 0; Logger.Error("No matching curly brace '{{' for this closing token.", token.Span); isBracketOk = false; } break; } return isBracketOk; }