public void Can_Without_LLK_Init()
{
var code = GetCode();
var lexer = new Lexer(code);
var tokens = lexer.Tokenize();
var tk = new TokenIterator();
tk.Init(tokens, 1, 0);
Assert.AreEqual(tk.LLK, -1);
Assert.AreEqual(tk.TokenList.Count, 36);
Assert.AreEqual(tk.CurrentIndex, -1);
Assert.IsNull(tk.NextToken);
Assert.AreEqual(tk.CurrentBatchIndex, -1);
tk.Advance();
Assert.AreEqual(tk.CurrentIndex, 0);
Assert.AreEqual(tk.CurrentBatchIndex, 0);
Assert.AreEqual(tk.NextToken.Token, Tokens.Var);
}
/// <summary>
/// Parses parameters.
/// </summary>
/// <param name="args">The list of arguments to store.</param>
/// <param name="tokenIt">The token iterator</param>
/// <param name="parser">The parser</param>
/// <param name="meta">The function meta for checking parameters</param>
/// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param>
/// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param>
public static void ParseFuncParameters(List <Expr> args, TokenIterator tokenIt, Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta)
{
var totalParameters = 0;
if (tokenIt.NextToken.Token == Tokens.LeftParenthesis)
{
expectParenthesis = true;
}
// START with check for "("
if (expectParenthesis)
{
tokenIt.Expect(Tokens.LeftParenthesis);
}
var passNewLine = !enableNewLineAsEnd;
var endTokens = BuildEndTokens(enableNewLineAsEnd, meta);
var totalNamedParams = 0;
var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0;
while (true)
{
Expr exp = null;
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
{
break;
}
if (tokenIt.NextToken.Token == Tokens.Comma)
{
tokenIt.Advance();
}
var token = tokenIt.NextToken.Token;
var peek = tokenIt.Peek().Token;
var isVar = parser.Context.Symbols.Contains(token.Text);
var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text);
var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch;
// CASE 1: Named params for external c# object method calls
// CASE 2: Named params for internal script functions ( where we have access to its param metadata )
if ((meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon) ||
(token.Kind == TokenKind.Ident && isParamNameMatch && !isVar) ||
(token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon) ||
(isKeywordParamName && !isVar))
{
var paramName = token.Text;
var namedParamToken = tokenIt.NextToken;
tokenIt.Advance();
// Advance and check if ":"
if (tokenIt.NextToken.Token == Tokens.Colon)
{
tokenIt.Advance();
}
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
exp = Exprs.NamedParam(paramName, exp, namedParamToken);
args.Add(exp);
totalNamedParams++;
}
// CASE 2: Name of variable being passed to function is same as one of the parameter names.
else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text))
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
{
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
}
var next = tokenIt.Peek();
if (next.Token.Kind == TokenKind.Symbol)
{
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false);
}
else
{
exp = parser.ParseIdExpression(null, null, false);
}
args.Add(exp);
}
// CASE 3: Normal param
else
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
{
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
}
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
args.Add(exp);
}
totalParameters++;
parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters);
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
{
break;
}
// Advance if not using fluent-parameters
if (meta == null)
{
tokenIt.Expect(Tokens.Comma);
}
}
// END with check for ")"
if (expectParenthesis)
{
tokenIt.Expect(Tokens.RightParenthesis);
}
}
/// <summary>
/// Parses an interpolated token into a set of tokens making up an interpolated expression.
/// </summary>
/// <param name="t"></param>
/// <returns></returns>
public Expr ParseInterpolatedExpression(Token t)
{
var iexp = new InterpolatedExpr();
iexp.Expressions = new List<Expr>();
// Convert each token in the interpolated string, need to convert it into
// it's own expression.
var tokens = t.Value as List<TokenData>;
foreach (var tokenData in tokens)
{
var token = tokenData.Token;
Expr exp = null;
// 1. true / false / "name" / 123 / null;
if (token.IsLiteralAny())
{
exp = token == Tokens.Null
? Exprs.Const(LObjects.Null, tokenData)
: Exprs.Const(TokenHelper.ConvertToLangLiteral(token), tokenData);
this.SetupContext(exp, tokenData);
_state.ExpressionCount++;
}
// 2. ${first + 'abc'} or ${ result / 2 + max }
else if (token.Kind == TokenKind.Multi)
{
var tokenIterator = new TokenIterator();
var tokens2 = token.Value as List<TokenData>;
tokenIterator.Init(tokens2, 1, 100);
tokenIterator.Advance();
var exisiting = _tokenIt;
// a. Temporarily set the token iterator for the parser to the one for the interpolation.
_tokenIt = tokenIterator;
SetupTokenIteratorReferences(this._tokenIt);
Exprs.Setup(_tokenIt, _context, _scriptPath);
// b. Now parse only the tokens supplied.
exp = ParseExpression(null);
// c. Reset the token iterator to the global one for the entire script.
_tokenIt = exisiting;
SetupTokenIteratorReferences(this._tokenIt);
Exprs.Setup(_tokenIt, _context, _scriptPath);
}
iexp.Expressions.Add(exp);
}
_tokenIt.Advance();
return iexp;
}
/// <summary>
/// Parses parameters.
/// </summary>
/// <param name="args">The list of arguments to store.</param>
/// <param name="tokenIt">The token iterator</param>
/// <param name="parser">The parser</param>
/// <param name="meta">The function meta for checking parameters</param>
/// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param>
/// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param>
public static void ParseFuncParameters(List<Expr> args, TokenIterator tokenIt, Parser.Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta)
{
int totalParameters = 0;
if (tokenIt.NextToken.Token == Tokens.LeftParenthesis)
expectParenthesis = true;
// START with check for "("
if (expectParenthesis) tokenIt.Expect(Tokens.LeftParenthesis);
bool passNewLine = !enableNewLineAsEnd;
var endTokens = BuildEndTokens(enableNewLineAsEnd, meta);
int totalNamedParams = 0;
var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0;
while (true)
{
Expr exp = null;
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
if (tokenIt.NextToken.Token == Tokens.Comma)
tokenIt.Advance();
var token = tokenIt.NextToken.Token;
var peek = tokenIt.Peek().Token;
var isVar = parser.Context.Symbols.Contains(token.Text);
var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text);
var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch;
// CASE 1: Named params for external c# object method calls
// CASE 2: Named params for internal script functions ( where we have access to its param metadata )
if ( (meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon )
|| (token.Kind == TokenKind.Ident && isParamNameMatch && !isVar)
|| (token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon)
|| (isKeywordParamName && !isVar ) )
{
var paramName = token.Text;
var namedParamToken = tokenIt.NextToken;
tokenIt.Advance();
// Advance and check if ":"
if (tokenIt.NextToken.Token == Tokens.Colon)
tokenIt.Advance();
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
exp = Exprs.NamedParam(paramName, exp, namedParamToken);
args.Add(exp);
totalNamedParams++;
}
// CASE 2: Name of variable being passed to function is same as one of the parameter names.
else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text))
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
var next = tokenIt.Peek();
if (next.Token.Kind == TokenKind.Symbol)
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false);
else
exp = parser.ParseIdExpression(null, null, false);
args.Add(exp);
}
// CASE 3: Normal param
else
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
args.Add(exp);
}
totalParameters++;
parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters);
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
// Advance if not using fluent-parameters
if(meta == null)
tokenIt.Expect(Tokens.Comma);
}
// END with check for ")"
if (expectParenthesis) tokenIt.Expect(Tokens.RightParenthesis);
}
/// <summary>
/// Match the current token to the token supplied.
/// </summary>
/// <param name="count">The number of positions to move forward</param>
/// <param name="passNewLine">Whether or not to pass a new line token</param>
protected void Advance(int count = 1, bool passNewLine = true)
{
_tokenIt.Advance(count, passNewLine);
}
