private ASTNode ParseInternal(string input)
{
tokenStream = new TokenStream(ExpressionTokenizer.Tokenize(input, StructList <ExpressionToken> .Get()));
expressionStack = expressionStack ?? StackPool <ASTNode> .Get();
operatorStack = operatorStack ?? StackPool <OperatorNode> .Get();
if (tokenStream.Current == ExpressionTokenType.ExpressionOpen)
{
tokenStream.Advance();
}
if (!tokenStream.HasMoreTokens)
{
throw new ParseException("Failed trying to parse empty expression");
}
if (tokenStream.Last == ExpressionTokenType.ExpressionClose)
{
tokenStream.Chop();
}
ASTNode retn = ParseLoop();
Release();
return(retn);
}
public TypeBodyNode Parse(string input, string fileName, int lineStart)
{
tokenStream = new TokenStream(ExpressionTokenizer.Tokenize(input, StructList <ExpressionToken> .Get()));
if (!tokenStream.HasMoreTokens)
{
throw new ParseException("Failed trying to parse empty expression");
}
TypeBodyNode retn = new TypeBodyNode();
int cnt = 0;
while (tokenStream.HasMoreTokens && cnt < 10000)
{
cnt++;
ExpressionToken current = tokenStream.Current;
ASTNode node = null;
if (ParseDeclaration(ref node))
{
retn.nodes.Add(node);
continue;
}
if (current == tokenStream.Current)
{
throw new ParseException($"Failed to parse {fileName}. Got stuck on {current.value}");
}
}
return(retn);
}
public void Tokenize_SingleToken_ReturnsExpectedToken(string path, ExpressionTokenType expectedType)
{
var tokenizer = new ExpressionTokenizer();
var token = tokenizer.Tokenize(path).Single();
Assert.AreEqual(expectedType, token.Type);
}
public void Tokenize_Operators()
{
string input = "+";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Plus, tokens[0].expressionTokenType);
input = "-";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
input = "*";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Times, tokens[0].expressionTokenType);
input = "/";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Divide, tokens[0].expressionTokenType);
input = "%";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Mod, tokens[0].expressionTokenType);
}
static IExpressionTreeNode <double> BuildExpressionTree(string expression, IVariableProvider <double> variableProvider)
{
var tokenizer = new ExpressionTokenizer(AbstractTokenParser.DefaultTokenParserChain);
var tokenizedExpression = tokenizer.Tokenize(expression);
var expressionTree = ExpressionTreeBuilder.BuildTree(tokenizedExpression, variableProvider);
return(expressionTree);
}
public void Tokenize_String()
{
string input = "'some string'";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual("some string", tokens[0].value);
}
public void AllowKeyWordAsIdentifierPart()
{
string input = "isThing";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Identifier);
AssertTokenTypes(types, tokens);
}
public void Tokenize_IllegalCharacters_ThrowsFormatException(string input)
{
var tokenizer = new ExpressionTokenizer();
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
Assert.Throws <FormatException>(() =>
{
tokenizer.Tokenize(input).ToList(); // Using ToList to force tokenization of the entire string
});
}
public void Tokenize_Null_ThrowsArgumentNullException()
{
var tokenizer = new ExpressionTokenizer();
Assert.Throws <ArgumentNullException>(() =>
{
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
tokenizer.Tokenize(null).ToList();
});
}
public void TokenizeBasicString()
{
string input = "item.thing";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(3, tokens.Count);
Assert.AreEqual("item", tokens[0].value);
Assert.AreEqual(".", tokens[1].value);
Assert.AreEqual("thing", tokens[2].value);
}
public void Tokenize_SpecialIdentifier()
{
string input = "1 + $ident";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Plus);
types.Add(ExpressionTokenType.Identifier);
AssertTokenTypes(types, tokens);
}
public void CreatesParenthesis(string str, TokenType tokenType)
{
// arrange
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Single(tokens);
Assert.Equal(new Token(tokenType), tokens[0]);
}
public void CreatesOperatorToken(string str, Operator @operator)
{
// arrange
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Single(tokens);
Assert.Equal(new Token(@operator), tokens[0]);
}
public void CreatesVariableToken()
{
// arrange
var str = "bananas";
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Single(tokens);
Assert.Equal(new Token("bananas"), tokens[0]);
}
public void Tokenize_ExpressionStatement()
{
string input = "{";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.ExpressionOpen, tokens[0].expressionTokenType);
input = "}";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.ExpressionClose, tokens[0].expressionTokenType);
}
public void CreatesValueToken()
{
// arrange
var str = "3.14159";
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Single(tokens);
Assert.Equal(new Token(float.Parse(str)), tokens[0]);
}
public void Tokenize_Boolean()
{
string input = "true";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual("true", tokens[0].value);
input = "false";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual("false", tokens[0].value);
}
public void Tokenize_ArrayAccess()
{
string input = "[";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.ArrayAccessOpen, tokens[0].expressionTokenType);
input = "]";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.ArrayAccessClose, tokens[0].expressionTokenType);
}
public void AllowKeyWordAsIdentifierPartInExpression()
{
string input = "isThing ? 1 : 2";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Identifier);
types.Add(ExpressionTokenType.QuestionMark);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Colon);
types.Add(ExpressionTokenType.Number);
AssertTokenTypes(types, tokens);
}
public static bool TryParseTypeName(string typeName, out TypeLookup typeLookup)
{
StructList <ExpressionToken> list = StructList <ExpressionToken> .Get();
ExpressionTokenizer.Tokenize(typeName, list);
ExpressionParser parser = new ExpressionParser(new TokenStream(list));
typeLookup = default;
bool valid = parser.ParseTypePath(ref typeLookup);
parser.Release();
list.Release();
return(valid);
}
public void HandlesMultipleTokens0()
{
// arrange
var str = "()";
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Equal(2, tokens.Count);
Assert.Equal(new Token(TokenType.LeftParenthesis), tokens[0]);
Assert.Equal(new Token(TokenType.RightParenthesis), tokens[1]);
}
public void Tokenize_Ternary()
{
string input = "value ? 1 : 2";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Identifier);
types.Add(ExpressionTokenType.QuestionMark);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Colon);
types.Add(ExpressionTokenType.Number);
AssertTokenTypes(types, tokens);
}
public void HandlesMultipleTokens1()
{
// arrange
var str = "(3.14159)";
var tokenizer = new ExpressionTokenizer();
// act
var tokens = tokenizer.Tokenize(str);
// assert
Assert.Equal(3, tokens.Count);
Assert.Equal(new Token(TokenType.LeftParenthesis), tokens[0]);
Assert.Equal(new Token(float.Parse("3.14159")), tokens[1]);
Assert.Equal(new Token(TokenType.RightParenthesis), tokens[2]);
}
public void Tokenize_Conditionals()
{
string input = "&&";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.AndAlso, tokens[0].expressionTokenType);
input = "||";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.OrElse, tokens[0].expressionTokenType);
input = "==";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Equals, tokens[0].expressionTokenType);
input = "!=";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.NotEquals, tokens[0].expressionTokenType);
input = ">";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.GreaterThan, tokens[0].expressionTokenType);
input = "<";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.LessThan, tokens[0].expressionTokenType);
input = ">=";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.GreaterThanEqualTo, tokens[0].expressionTokenType);
input = "<=";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.LessThanEqualTo, tokens[0].expressionTokenType);
input = "!";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Not, tokens[0].expressionTokenType);
}
public void Tokenize_CompoundArrayAccess()
{
string input = "366 + something[first]second.third";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(9, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Number, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Plus, tokens[1].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Identifier, tokens[2].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.ArrayAccessOpen, tokens[3].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Identifier, tokens[4].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.ArrayAccessClose, tokens[5].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Identifier, tokens[6].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Dot, tokens[7].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Identifier, tokens[8].expressionTokenType);
}
public void Tokenize_Comma()
{
string input = "method(1, 2, 3)";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Identifier);
types.Add(ExpressionTokenType.ParenOpen);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Comma);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Comma);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.ParenClose);
AssertTokenTypes(types, tokens);
}
public void Tokenize_CompoundOperatorExpression()
{
string input = "52 + 2.4";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(3, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Number, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Plus, tokens[1].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[2].expressionTokenType);
input = "-52 * 714";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(4, tokens.Count);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[1].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Times, tokens[2].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[3].expressionTokenType);
}
public void Tokenize_Number()
{
string input = "6264.1";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual("6264.1", tokens[0].value);
Assert.AreEqual(ExpressionTokenType.Number, tokens[0].expressionTokenType);
input = "-6264.1";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual("-", tokens[0].value);
Assert.AreEqual("6264.1", tokens[1].value);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[1].expressionTokenType);
input = "-6264";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual("-", tokens[0].value);
Assert.AreEqual("6264", tokens[1].value);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[1].expressionTokenType);
input = "-6264f";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual("-", tokens[0].value);
Assert.AreEqual("6264f", tokens[1].value);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[1].expressionTokenType);
input = "-6264.414f ";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual("-", tokens[0].value);
Assert.AreEqual("6264.414f", tokens[1].value);
Assert.AreEqual(ExpressionTokenType.Minus, tokens[0].expressionTokenType);
Assert.AreEqual(ExpressionTokenType.Number, tokens[1].expressionTokenType);
input = "6264";
tokens = ExpressionTokenizer.Tokenize(input);
Assert.AreEqual(1, tokens.Count);
Assert.AreEqual("6264", tokens[0].value);
Assert.AreEqual(ExpressionTokenType.Number, tokens[0].expressionTokenType);
}
private RPNExpression ParseToRPN(string s, TokenPosition position)
{
RPNExpression rpnExpression = new RPNExpression(_functionEvaluator);
rpnExpression.Start();
ExpressionToken[] tokens = _tokenizer.Tokenize(s, position);
foreach (ExpressionToken token in tokens)
{
if (token.TokenType != TokenType.WhiteSpace)
{
rpnExpression.ApplyToken(token);
}
}
rpnExpression.Finish();
return(rpnExpression);
}
public void Tokenize_ComplexUnary()
{
string input = "item != 55 && !someCondition || -(11 * 4)";
StructList <ExpressionToken> tokens = ExpressionTokenizer.Tokenize(input);
List <ExpressionTokenType> types = new List <ExpressionTokenType>();
types.Add(ExpressionTokenType.Identifier);
types.Add(ExpressionTokenType.NotEquals);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.AndAlso);
types.Add(ExpressionTokenType.Not);
types.Add(ExpressionTokenType.Identifier);
types.Add(ExpressionTokenType.OrElse);
types.Add(ExpressionTokenType.Minus);
types.Add(ExpressionTokenType.ParenOpen);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.Times);
types.Add(ExpressionTokenType.Number);
types.Add(ExpressionTokenType.ParenClose);
AssertTokenTypes(types, tokens);
}
