public void Can_Without_LLK_Init() { var code = GetCode(); var lexer = new Lexer(code); var tokens = lexer.Tokenize(); var tk = new TokenIterator(); tk.Init(tokens, 1, 0); Assert.AreEqual(tk.LLK, -1); Assert.AreEqual(tk.TokenList.Count, 36); Assert.AreEqual(tk.CurrentIndex, -1); Assert.IsNull(tk.NextToken); Assert.AreEqual(tk.CurrentBatchIndex, -1); tk.Advance(); Assert.AreEqual(tk.CurrentIndex, 0); Assert.AreEqual(tk.CurrentBatchIndex, 0); Assert.AreEqual(tk.NextToken.Token, Tokens.Var); }
/// <summary> /// Parses parameters. /// </summary> /// <param name="args">The list of arguments to store.</param> /// <param name="tokenIt">The token iterator</param> /// <param name="parser">The parser</param> /// <param name="meta">The function meta for checking parameters</param> /// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param> /// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param> public static void ParseFuncParameters(List <Expr> args, TokenIterator tokenIt, Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta) { var totalParameters = 0; if (tokenIt.NextToken.Token == Tokens.LeftParenthesis) { expectParenthesis = true; } // START with check for "(" if (expectParenthesis) { tokenIt.Expect(Tokens.LeftParenthesis); } var passNewLine = !enableNewLineAsEnd; var endTokens = BuildEndTokens(enableNewLineAsEnd, meta); var totalNamedParams = 0; var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0; while (true) { Expr exp = null; // Check for end of statment or invalid end of script. if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd)) { break; } if (tokenIt.NextToken.Token == Tokens.Comma) { tokenIt.Advance(); } var token = tokenIt.NextToken.Token; var peek = tokenIt.Peek().Token; var isVar = parser.Context.Symbols.Contains(token.Text); var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text); var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch; // CASE 1: Named params for external c# object method calls // CASE 2: Named params for internal script functions ( where we have access to its param metadata ) if ((meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon) || (token.Kind == TokenKind.Ident && isParamNameMatch && !isVar) || (token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon) || (isKeywordParamName && !isVar)) { var paramName = token.Text; var namedParamToken = tokenIt.NextToken; tokenIt.Advance(); // Advance and check if ":" if (tokenIt.NextToken.Token == Tokens.Colon) { tokenIt.Advance(); } exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true); exp = Exprs.NamedParam(paramName, exp, namedParamToken); args.Add(exp); totalNamedParams++; } // CASE 2: Name of variable being passed to function is same as one of the parameter names. else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text)) { // Can not have normal parameters after named parameters. if (totalNamedParams > 0) { throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters"); } var next = tokenIt.Peek(); if (next.Token.Kind == TokenKind.Symbol) { exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false); } else { exp = parser.ParseIdExpression(null, null, false); } args.Add(exp); } // CASE 3: Normal param else { // Can not have normal parameters after named parameters. if (totalNamedParams > 0) { throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters"); } exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true); args.Add(exp); } totalParameters++; parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters); // Check for end of statment or invalid end of script. if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd)) { break; } // Advance if not using fluent-parameters if (meta == null) { tokenIt.Expect(Tokens.Comma); } } // END with check for ")" if (expectParenthesis) { tokenIt.Expect(Tokens.RightParenthesis); } }
/// <summary> /// Parses an interpolated token into a set of tokens making up an interpolated expression. /// </summary> /// <param name="t"></param> /// <returns></returns> public Expr ParseInterpolatedExpression(Token t) { var iexp = new InterpolatedExpr(); iexp.Expressions = new List<Expr>(); // Convert each token in the interpolated string, need to convert it into // it's own expression. var tokens = t.Value as List<TokenData>; foreach (var tokenData in tokens) { var token = tokenData.Token; Expr exp = null; // 1. true / false / "name" / 123 / null; if (token.IsLiteralAny()) { exp = token == Tokens.Null ? Exprs.Const(LObjects.Null, tokenData) : Exprs.Const(TokenHelper.ConvertToLangLiteral(token), tokenData); this.SetupContext(exp, tokenData); _state.ExpressionCount++; } // 2. ${first + 'abc'} or ${ result / 2 + max } else if (token.Kind == TokenKind.Multi) { var tokenIterator = new TokenIterator(); var tokens2 = token.Value as List<TokenData>; tokenIterator.Init(tokens2, 1, 100); tokenIterator.Advance(); var exisiting = _tokenIt; // a. Temporarily set the token iterator for the parser to the one for the interpolation. _tokenIt = tokenIterator; SetupTokenIteratorReferences(this._tokenIt); Exprs.Setup(_tokenIt, _context, _scriptPath); // b. Now parse only the tokens supplied. exp = ParseExpression(null); // c. Reset the token iterator to the global one for the entire script. _tokenIt = exisiting; SetupTokenIteratorReferences(this._tokenIt); Exprs.Setup(_tokenIt, _context, _scriptPath); } iexp.Expressions.Add(exp); } _tokenIt.Advance(); return iexp; }
/// <summary> /// Parses parameters. /// </summary> /// <param name="args">The list of arguments to store.</param> /// <param name="tokenIt">The token iterator</param> /// <param name="parser">The parser</param> /// <param name="meta">The function meta for checking parameters</param> /// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param> /// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param> public static void ParseFuncParameters(List<Expr> args, TokenIterator tokenIt, Parser.Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta) { int totalParameters = 0; if (tokenIt.NextToken.Token == Tokens.LeftParenthesis) expectParenthesis = true; // START with check for "(" if (expectParenthesis) tokenIt.Expect(Tokens.LeftParenthesis); bool passNewLine = !enableNewLineAsEnd; var endTokens = BuildEndTokens(enableNewLineAsEnd, meta); int totalNamedParams = 0; var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0; while (true) { Expr exp = null; // Check for end of statment or invalid end of script. if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd)) break; if (tokenIt.NextToken.Token == Tokens.Comma) tokenIt.Advance(); var token = tokenIt.NextToken.Token; var peek = tokenIt.Peek().Token; var isVar = parser.Context.Symbols.Contains(token.Text); var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text); var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch; // CASE 1: Named params for external c# object method calls // CASE 2: Named params for internal script functions ( where we have access to its param metadata ) if ( (meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon ) || (token.Kind == TokenKind.Ident && isParamNameMatch && !isVar) || (token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon) || (isKeywordParamName && !isVar ) ) { var paramName = token.Text; var namedParamToken = tokenIt.NextToken; tokenIt.Advance(); // Advance and check if ":" if (tokenIt.NextToken.Token == Tokens.Colon) tokenIt.Advance(); exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true); exp = Exprs.NamedParam(paramName, exp, namedParamToken); args.Add(exp); totalNamedParams++; } // CASE 2: Name of variable being passed to function is same as one of the parameter names. else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text)) { // Can not have normal parameters after named parameters. if (totalNamedParams > 0) throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters"); var next = tokenIt.Peek(); if (next.Token.Kind == TokenKind.Symbol) exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false); else exp = parser.ParseIdExpression(null, null, false); args.Add(exp); } // CASE 3: Normal param else { // Can not have normal parameters after named parameters. if (totalNamedParams > 0) throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters"); exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true); args.Add(exp); } totalParameters++; parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters); // Check for end of statment or invalid end of script. if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd)) break; // Advance if not using fluent-parameters if(meta == null) tokenIt.Expect(Tokens.Comma); } // END with check for ")" if (expectParenthesis) tokenIt.Expect(Tokens.RightParenthesis); }
/// <summary> /// Match the current token to the token supplied. /// </summary> /// <param name="count">The number of positions to move forward</param> /// <param name="passNewLine">Whether or not to pass a new line token</param> protected void Advance(int count = 1, bool passNewLine = true) { _tokenIt.Advance(count, passNewLine); }