private TokenIterator GetTokenIterator(int llK, string code) { var lexer = new Lexer(code); var tk = new TokenIterator(); tk.Init((llk) => lexer.GetTokenBatch(llk), llK, (pos) => lexer.ResetPos(pos)); return(tk); }
public void Can_Without_LLK_Init() { var code = GetCode(); var lexer = new Lexer(code); var tokens = lexer.Tokenize(); var tk = new TokenIterator(); tk.Init(tokens, 1, 0); Assert.AreEqual(tk.LLK, -1); Assert.AreEqual(tk.TokenList.Count, 36); Assert.AreEqual(tk.CurrentIndex, -1); Assert.IsNull(tk.NextToken); Assert.AreEqual(tk.CurrentBatchIndex, -1); tk.Advance(); Assert.AreEqual(tk.CurrentIndex, 0); Assert.AreEqual(tk.CurrentBatchIndex, 0); Assert.AreEqual(tk.NextToken.Token, Tokens.Var); }
/// <summary> /// Parses an interpolated token into a set of tokens making up an interpolated expression. /// </summary> /// <param name="t"></param> /// <returns></returns> public Expr ParseInterpolatedExpression(Token t) { var iexp = new InterpolatedExpr(); iexp.Expressions = new List<Expr>(); // Convert each token in the interpolated string, need to convert it into // it's own expression. var tokens = t.Value as List<TokenData>; foreach (var tokenData in tokens) { var token = tokenData.Token; Expr exp = null; // 1. true / false / "name" / 123 / null; if (token.IsLiteralAny()) { exp = token == Tokens.Null ? Exprs.Const(LObjects.Null, tokenData) : Exprs.Const(TokenHelper.ConvertToLangLiteral(token), tokenData); this.SetupContext(exp, tokenData); _state.ExpressionCount++; } // 2. ${first + 'abc'} or ${ result / 2 + max } else if (token.Kind == TokenKind.Multi) { var tokenIterator = new TokenIterator(); var tokens2 = token.Value as List<TokenData>; tokenIterator.Init(tokens2, 1, 100); tokenIterator.Advance(); var exisiting = _tokenIt; // a. Temporarily set the token iterator for the parser to the one for the interpolation. _tokenIt = tokenIterator; SetupTokenIteratorReferences(this._tokenIt); Exprs.Setup(_tokenIt, _context, _scriptPath); // b. Now parse only the tokens supplied. exp = ParseExpression(null); // c. Reset the token iterator to the global one for the entire script. _tokenIt = exisiting; SetupTokenIteratorReferences(this._tokenIt); Exprs.Setup(_tokenIt, _context, _scriptPath); } iexp.Expressions.Add(exp); } _tokenIt.Advance(); return iexp; }