public void SingleQuoteTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.SYMBOL, tokens.Peek(1).GetType());
Assert.AreEqual("\"", tokens.Peek(1).GetValue().ToString());
}
public void ComplexEmbeddingTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\"<a(href=\"http://www.microsoft.com\") \"Microsoft Corp\">\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Test token
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.EMBEDDING, tokens.Peek(1).GetType());
//Test tokens in embedding
EmbeddingToken embeddingToken = (EmbeddingToken)tokens.NextToken();
TokenIterator embeddingTokens = embeddingToken.GetTokenIterator();
Assert.AreEqual(12, embeddingTokens.GetSize());
Assert.AreEqual("\"", embeddingTokens.Peek(1).GetValue().ToString());
Assert.AreEqual("", embeddingTokens.Peek(2).GetValue().ToString());
Assert.AreEqual("<", embeddingTokens.Peek(3).GetValue().ToString());
Assert.AreEqual("a", embeddingTokens.Peek(4).GetValue().ToString());
Assert.AreEqual("(", embeddingTokens.Peek(5).GetValue().ToString());
Assert.AreEqual("href", embeddingTokens.Peek(6).GetValue().ToString());
Assert.AreEqual("=", embeddingTokens.Peek(7).GetValue().ToString());
Assert.AreEqual("http://www.microsoft.com", embeddingTokens.Peek(8).GetValue().ToString());
Assert.AreEqual(")", embeddingTokens.Peek(9).GetValue().ToString());
Assert.AreEqual("Microsoft Corp", embeddingTokens.Peek(10).GetValue().ToString());
Assert.AreEqual(">", embeddingTokens.Peek(11).GetValue().ToString());
Assert.AreEqual("\"", embeddingTokens.Peek(12).GetValue().ToString());
}
public void EmbeddingTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\"pre<\"\\\">\">post\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Test token
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.EMBEDDING, tokens.Peek(1).GetType());
//Get embedding and test inner tokens
EmbeddingToken parsedToken = (EmbeddingToken)tokens.NextToken();
TokenIterator embeddingTokens = parsedToken.GetTokenIterator();
Assert.AreEqual(7, embeddingTokens.GetSize());
Assert.AreEqual("\"", embeddingTokens.Peek(1).GetValue().ToString());
Assert.AreEqual("pre", embeddingTokens.Peek(2).GetValue().ToString());
Assert.AreEqual("<", embeddingTokens.Peek(3).GetValue().ToString());
Assert.AreEqual("\\\">", embeddingTokens.Peek(4).GetValue().ToString());
Assert.AreEqual(">", embeddingTokens.Peek(5).GetValue().ToString());
Assert.AreEqual("post", embeddingTokens.Peek(6).GetValue().ToString());
Assert.AreEqual("\"", embeddingTokens.Peek(7).GetValue().ToString());
}
public void PeekTest()
{
//Create list and add dummy tokens to it
List <Token> tokenStream = new List <Token>();
Token token1 = new Token();
Token token2 = new Token();
tokenStream.Add(token1);
tokenStream.Add(token2);
//Create iterator with tokenstream and then test some things out
TokenIterator iterator = new TokenIterator(tokenStream);
//Test Peek method
Assert.AreEqual(token1, iterator.Peek(1));
Assert.AreEqual(token2, iterator.Peek(2));
}
public void PeekTest()
{
List <Token> tokenStream = null; // TODO: Initialize to an appropriate value
TokenIterator target = new TokenIterator(tokenStream); // TODO: Initialize to an appropriate value
int offset = 0; // TODO: Initialize to an appropriate value
Token expected = null; // TODO: Initialize to an appropriate value
Token actual;
actual = target.Peek(offset);
Assert.AreEqual(expected, actual);
Assert.Inconclusive("Verify the correctness of this test method.");
}
/// <summary>
/// Parses parameters.
/// </summary>
/// <param name="args">The list of arguments to store.</param>
/// <param name="tokenIt">The token iterator</param>
/// <param name="parser">The parser</param>
/// <param name="meta">The function meta for checking parameters</param>
/// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param>
/// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param>
public static void ParseFuncParameters(List <Expr> args, TokenIterator tokenIt, Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta)
{
var totalParameters = 0;
if (tokenIt.NextToken.Token == Tokens.LeftParenthesis)
{
expectParenthesis = true;
}
// START with check for "("
if (expectParenthesis)
{
tokenIt.Expect(Tokens.LeftParenthesis);
}
var passNewLine = !enableNewLineAsEnd;
var endTokens = BuildEndTokens(enableNewLineAsEnd, meta);
var totalNamedParams = 0;
var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0;
while (true)
{
Expr exp = null;
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
{
break;
}
if (tokenIt.NextToken.Token == Tokens.Comma)
{
tokenIt.Advance();
}
var token = tokenIt.NextToken.Token;
var peek = tokenIt.Peek().Token;
var isVar = parser.Context.Symbols.Contains(token.Text);
var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text);
var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch;
// CASE 1: Named params for external c# object method calls
// CASE 2: Named params for internal script functions ( where we have access to its param metadata )
if ((meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon) ||
(token.Kind == TokenKind.Ident && isParamNameMatch && !isVar) ||
(token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon) ||
(isKeywordParamName && !isVar))
{
var paramName = token.Text;
var namedParamToken = tokenIt.NextToken;
tokenIt.Advance();
// Advance and check if ":"
if (tokenIt.NextToken.Token == Tokens.Colon)
{
tokenIt.Advance();
}
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
exp = Exprs.NamedParam(paramName, exp, namedParamToken);
args.Add(exp);
totalNamedParams++;
}
// CASE 2: Name of variable being passed to function is same as one of the parameter names.
else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text))
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
{
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
}
var next = tokenIt.Peek();
if (next.Token.Kind == TokenKind.Symbol)
{
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false);
}
else
{
exp = parser.ParseIdExpression(null, null, false);
}
args.Add(exp);
}
// CASE 3: Normal param
else
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
{
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
}
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
args.Add(exp);
}
totalParameters++;
parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters);
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
{
break;
}
// Advance if not using fluent-parameters
if (meta == null)
{
tokenIt.Expect(Tokens.Comma);
}
}
// END with check for ")"
if (expectParenthesis)
{
tokenIt.Expect(Tokens.RightParenthesis);
}
}
/// <summary>
/// Parses parameters.
/// </summary>
/// <param name="args">The list of arguments to store.</param>
/// <param name="tokenIt">The token iterator</param>
/// <param name="parser">The parser</param>
/// <param name="meta">The function meta for checking parameters</param>
/// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param>
/// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param>
public static void ParseFuncParameters(List<Expr> args, TokenIterator tokenIt, Parser.Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta)
{
int totalParameters = 0;
if (tokenIt.NextToken.Token == Tokens.LeftParenthesis)
expectParenthesis = true;
// START with check for "("
if (expectParenthesis) tokenIt.Expect(Tokens.LeftParenthesis);
bool passNewLine = !enableNewLineAsEnd;
var endTokens = BuildEndTokens(enableNewLineAsEnd, meta);
int totalNamedParams = 0;
var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0;
while (true)
{
Expr exp = null;
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
if (tokenIt.NextToken.Token == Tokens.Comma)
tokenIt.Advance();
var token = tokenIt.NextToken.Token;
var peek = tokenIt.Peek().Token;
var isVar = parser.Context.Symbols.Contains(token.Text);
var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text);
var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch;
// CASE 1: Named params for external c# object method calls
// CASE 2: Named params for internal script functions ( where we have access to its param metadata )
if ( (meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon )
|| (token.Kind == TokenKind.Ident && isParamNameMatch && !isVar)
|| (token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon)
|| (isKeywordParamName && !isVar ) )
{
var paramName = token.Text;
var namedParamToken = tokenIt.NextToken;
tokenIt.Advance();
// Advance and check if ":"
if (tokenIt.NextToken.Token == Tokens.Colon)
tokenIt.Advance();
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
exp = Exprs.NamedParam(paramName, exp, namedParamToken);
args.Add(exp);
totalNamedParams++;
}
// CASE 2: Name of variable being passed to function is same as one of the parameter names.
else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text))
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
var next = tokenIt.Peek();
if (next.Token.Kind == TokenKind.Symbol)
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false);
else
exp = parser.ParseIdExpression(null, null, false);
args.Add(exp);
}
// CASE 3: Normal param
else
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
args.Add(exp);
}
totalParameters++;
parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters);
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
// Advance if not using fluent-parameters
if(meta == null)
tokenIt.Expect(Tokens.Comma);
}
// END with check for ")"
if (expectParenthesis) tokenIt.Expect(Tokens.RightParenthesis);
}
public virtual TokenData Peek(int count)
{
return(_tokenIt.Peek(count));
}
/// <summary>
/// Peek at the token ahead of the current token
/// </summary>
/// <param name="count">The number of tokens ahead to peek at</param>
/// <param name="passNewLine">Whether or not the pass a new line token when peeking</param>
protected TokenData Peek(int count = 1, bool passNewLine = false)
{
return(_tokenIt.Peek(1, passNewLine));
}
