public void Tokenize_WithUndefinedSymbol_ThrowsExceptionWithMessage()
{
//ARRANGE
string source = "[[]";
//ACT
//ASSERT
var ex = Assert.Throws <Exception>(() => _lexer.Tokenize(source).ToList());
Assert.That(ex.Message, Is.EqualTo("Unrecognized symbol '['."));
}
public void TestMethod3()
{
Lexer.Lexer lexer = new Lexer.Lexer();
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex("[Aa][Dd][Dd]"), Type = TokenTyp.Add
});
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex("\\("), Type = TokenTyp.OpenParenthesis
});
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex("\\)"), Type = TokenTyp.CloseParenthesis
});
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex(","), Type = TokenTyp.Comma
});
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex(@"\d+"), Type = TokenTyp.Number
});
lexer.AddDefinition(new TokenDefinition {
Regex = new Regex(@"\s+"), Type = TokenTyp.Whitespace, IsIgnored = true
});
LanguageParser parser = new LanguageParser();
int res = parser.Parse(lexer.Tokenize("ADD(ADD(1,ADD(1,3)) ,ADD(1,5)").ToList());
Assert.AreEqual(res, 11);
}
/// <summary>
/// コンパイルするコードを指定してコンパイルし、実行します。
/// </summary>
/// <param name="code">コンパイルされるコード文字列。</param>
/// <returns>コンパイルに成功したとき true、それ以外のとき false。</returns>
public LuryObject Evaluate(string code)
{
var lexer = new Lexer.Lexer(code + '\n');
var succeedTokenize = lexer.Tokenize();
lexer.Logger.CopyTo(this.OutputLogger);
if (!succeedTokenize)
{
return(null);
}
var globalContext = new LuryContext();
Intrinsic.SetBuiltInFunctions(globalContext);
var parser = new Parser();
Routine routine = null;
// parsing
try
{
routine = (Routine)parser.yyparse(new Lex2yyInput(lexer), new yydebug.yyDebugSimple());
}
catch (yyParser.yyException ex)
{
this.ReportyyException(ex, code);
return(null);
}
// running
if (routine == null)
{
return(null);
}
else
{
try
{
var exit = routine.Evaluate(globalContext);
if (exit.ExitReason == StatementExitReason.Break)
{
throw new LuryException(LuryExceptionType.WrongBreak);
}
return(exit.ReturnValue);
}
catch (LuryException ex)
{
Console.Error.WriteLine(ex.Message);
return(null);
}
}
}
public void Tokenize__ConstantSum__Correct()
{
var expr = "int.MaxValue + long.MaxValue";
var lexer = new Lexer.Lexer(expr);
var tokens = lexer.Tokenize();
Assert.Equal(TokenType.Constant, tokens[0].Type);
Assert.Equal(int.MaxValue.ToString(), tokens[0].Value);
Assert.Equal(TokenType.Plus, tokens[1].Type);
Assert.Equal(TokenType.Constant, tokens[2].Type);
Assert.Equal(long.MaxValue.ToString(), tokens[2].Value);
}
public void Tokenize__ConstantMulWithExcessBrackets__Correct()
{
var expr = "(int.MaxValue + (long.MaxValue))*(int.MinValue*long.MinValue)";
var lexer = new Lexer.Lexer(expr);
var tokens = lexer.Tokenize();
Assert.Equal(TokenType.Constant, tokens[1].Type);
Assert.Equal(TokenType.Constant, tokens[4].Type);
Assert.Equal(TokenType.Minus, tokens[9].Type);
Assert.Equal(TokenType.Constant, tokens[10].Type);
Assert.Equal(TokenType.Minus, tokens[12].Type);
Assert.Equal(TokenType.Constant, tokens[13].Type);
}
public void Tokenize__ConstantMul__Correct()
{
var expr = "(int.MaxValue + long.MaxValue)*(int.MinValue*long.MinValue)";
var lexer = new Lexer.Lexer(expr);
var tokens = lexer.Tokenize();
Assert.Equal(TokenType.Constant, tokens[1].Type);
Assert.Equal(TokenType.Constant, tokens[3].Type);
Assert.Equal(TokenType.Minus, tokens[7].Type);
Assert.Equal(TokenType.Constant, tokens[8].Type);
Assert.Equal(TokenType.Minus, tokens[10].Type);
Assert.Equal(TokenType.Constant, tokens[11].Type);
}
public static IStatement[] GetParseResultStatements(string expression, MethodInfo[] methodInfos = null)
{
var lexer = new Lexer.Lexer(expression);
var readOnlyList = lexer.Tokenize();
var context = new ParserContext(
readOnlyList,
new Dictionary <string, CompilerType>
{
{ "x", CompilerType.Long }, { "y", CompilerType.Long }, { "z", CompilerType.Long }
},
new Dictionary <string, FieldInfo>(),
methodInfos?.ToDictionary(x => x.Name, x => x) ?? new Dictionary <string, MethodInfo>(),
true);
var parser = new Parser.Parser(context);
var result = parser.Parse();
return(result.Statements);
}
public void Tokenize__BracketNumBracket__LexerDoesntReturnMethodCallExpression()
{
var randomExpr = @"x *(8 * (5))";
var lexer = new Lexer.Lexer(randomExpr);
var tokens = lexer.Tokenize();
Assert.Equal(tokens[0].Type, TokenType.Variable);
Assert.Equal(tokens[0].Value, "x");
Assert.Equal(tokens[1].Type, TokenType.Star);
Assert.Equal(tokens[2].Type, TokenType.LeftParent);
Assert.Equal(tokens[3].Type, TokenType.Constant);
Assert.Equal(tokens[3].Value, "8");
Assert.Equal(tokens[4].Type, TokenType.Star);
Assert.Equal(tokens[5].Type, TokenType.LeftParent);
Assert.Equal(tokens[6].Type, TokenType.Constant);
Assert.Equal(tokens[6].Value, "5");
Assert.Equal(tokens[7].Type, TokenType.RightParent);
Assert.Equal(tokens[8].Type, TokenType.RightParent);
}
public static IExpression GetParseResultExpression(string expression, bool constantFolding = true,
MethodInfo[] methodInfos = null)
{
var lexer = new Lexer.Lexer(expression);
var readOnlyList = lexer.Tokenize();
var context = new ParserContext(
readOnlyList,
new Dictionary <string, CompilerType>
{
{ "x", CompilerType.Long }, { "y", CompilerType.Long }, { "z", CompilerType.Long }
},
null,
methodInfos?.ToDictionary(x => x.Name, x => x) ?? new Dictionary <string, MethodInfo>(),
constantFolding
);
var parser = new Parser.Parser(context);
var result = parser.ParseExpression();
return(result);
}
public void Tokenize__IfElseStatement__Correct()
{
var expr = "if(1 == 1) {return 1} else {return 2}";
var lexer = new Lexer.Lexer(expr);
var result = lexer.Tokenize();
Assert.Equal(TokenType.IfWord, result[0].Type);
Assert.Equal(TokenType.LeftParent, result[1].Type);
Assert.Equal(TokenType.Constant, result[2].Type);
Assert.Equal(TokenType.EqualTo, result[3].Type);
Assert.Equal(TokenType.Constant, result[4].Type);
Assert.Equal(TokenType.RightParent, result[5].Type);
Assert.Equal(TokenType.LeftBrace, result[6].Type);
Assert.Equal(TokenType.ReturnWord, result[7].Type);
Assert.Equal(TokenType.Constant, result[8].Type);
Assert.Equal(TokenType.RightBrace, result[9].Type);
Assert.Equal(TokenType.ElseWord, result[10].Type);
Assert.Equal(TokenType.LeftBrace, result[11].Type);
Assert.Equal(TokenType.ReturnWord, result[12].Type);
Assert.Equal(TokenType.Constant, result[13].Type);
Assert.Equal(TokenType.RightBrace, result[14].Type);
}
public void ParseAxisParameter_WithSubsequentIdentifiers_SucceedsReturnsAxisParameter()
{
//ARRANGE
const string queryString = "[Aaa].[Bbb].[Ccc].FUNCTION(1, 2).FUNCTION";
const string expectedString = "[Aaa].[Bbb].[Ccc].FUNCTION(1, 2).FUNCTION";
//ACT
MdxExpressionBase expression;
bool isSucceeded = MdxParser.TryParseMember(_lexer.Tokenize(queryString).GetStatedTwoWayEnumerator(), out expression);
//ASSERT
Assert.That(isSucceeded, Is.True);
Assert.That(expression, Is.InstanceOf <MdxMember>());
Assert.That(expression.ToString(), Is.EqualTo(expectedString));
}
private IReadOnlyList <Token> GetLexerResult(string expr)
{
var lexer = new Lexer.Lexer(expr);
return(lexer.Tokenize());
}
private ExpressionNode ParseInterpolatedString()
{
Expect(TokenType.INTERPOLATED_STRING);
var pos = Position();
var str = Current().Text;
Next();
var parts = new List <ExpressionNode>();
bool inExpr = false;
int level = 0;
var start = 0;
int i = 0;
char peek() => str[i];
string current() => str.Substring(start, i - start - 1);
bool accept(char c)
{
if (peek() == c)
{
i++;
return(true);
}
return(false);
}
void addString()
{
var cur = current();
parts.Add(new StringNode(new SourcePosition(pos.Line, pos.Column + i - cur.Length + 1), cur));
}
void addExpression()
{
var cur = current();
var lexer = new Lexer.Lexer(cur, new SourcePosition(pos.Line - 1, 0 /* @TODO wrong column number */));
var parser = new Parser("<interpolated string expression>", lexer.Tokenize());
parts.Add(parser.ParseExpression());
}
while (i < str.Length)
{
if (inExpr)
{
if (accept('{'))
{
level++;
}
else if (accept('}'))
{
level--;
}
if (level == 0)
{
addExpression();
inExpr = false;
start = i;
}
else
{
i++;
}
}
else
{
if (accept('{'))
{
addString();
inExpr = true;
level = 1;
start = i;
}
else
{
i++;
}
}
}
if (inExpr)
{
throw new ParseException("String interpolation expression not ended");
}
if (start != i)
{
i++;
addString();
}
if (parts.Count == 1)
{
return(parts[0]);
}
else
{
var ret = new BinaryNode(pos /* @TODO: wrong column number */, BinaryOP.CONCAT, parts[0], parts[1]);
for (var j = 2; j < parts.Count; j++)
{
ret = new BinaryNode(pos /* @TODO: wrong column number */, BinaryOP.CONCAT, ret, parts[j]);
}
return(ret);
}
}
private void LexerTests()
{
Lexer.Lexer lex = new Lexer.Lexer();
List <Token> tokens = lex.Tokenize("//uibutton");
Debug.Assert(tokens.Count == 2);
Debug.Assert(tokens [0].Content == "//");
Debug.Assert(tokens [0].Desc == "ANCESTOR");
Debug.Assert(tokens [1].Content == "uibutton");
Debug.Assert(tokens [1].Desc == "IDENTIFIER");
tokens = lex.Tokenize("/uibutton");
Debug.Assert(tokens.Count == 2);
Debug.Assert(tokens [0].Content == "/");
Debug.Assert(tokens [0].Desc == "CHILD");
Debug.Assert(tokens [1].Content == "uibutton");
Debug.Assert(tokens [1].Desc == "IDENTIFIER");
tokens = lex.Tokenize("//uilabel[3]");
Debug.Assert(tokens.Count == 5);
Debug.Assert(tokens [0].Content == "//");
Debug.Assert(tokens [1].Content == "uilabel");
Debug.Assert(tokens [2].Content == "[");
Debug.Assert(tokens [3].Content == "3");
Debug.Assert(tokens [4].Content == "]");
Debug.Assert(tokens [2].Desc == "OPEN_PREDICATE");
Debug.Assert(tokens [3].Desc == "NUMBER");
tokens = lex.Tokenize("/ uibutton");
Debug.Assert(tokens.Count == 3);
Debug.Assert(tokens [0].Content == "/");
Debug.Assert(tokens [0].Desc == "CHILD");
Debug.Assert(tokens [2].Content == "uibutton");
Debug.Assert(tokens [1].Content == " ");
Debug.Assert(tokens [1].Desc == "WHITE_SPACE");
Debug.Assert(tokens [2].Desc == "IDENTIFIER");
tokens = lex.Tokenize("//uilabel/uibutton//altceva");
Debug.Assert(tokens.Count == 6);
Debug.Assert(tokens [3].Content == "uibutton");
Debug.Assert(tokens [3].Desc == "IDENTIFIER");
Debug.Assert(tokens [2].Desc == "CHILD");
tokens = lex.Tokenize("//label[@text=\"ceva\"]");
Debug.Assert(tokens.Count == 7);
Debug.Assert(tokens [3].Content == "text");
Debug.Assert(tokens [3].Desc == "ATTRIBUTE");
Debug.Assert(tokens [4].Content == "=");
Debug.Assert(tokens [4].Desc == "EQUAL");
Debug.Assert(tokens [5].Content == "ceva");
Debug.Assert(tokens [5].Desc == "STRING");
tokens = lex.Tokenize("//label[@text=\"ceva\" and @tre=\"23232\"]");
Debug.Assert(tokens.Count == 13);
Debug.Assert(tokens [3].Content == "text");
Debug.Assert(tokens [3].Desc == "ATTRIBUTE");
Debug.Assert(tokens [4].Content == "=");
Debug.Assert(tokens [4].Desc == "EQUAL");
Debug.Assert(tokens [5].Content == "ceva");
Debug.Assert(tokens [5].Desc == "STRING");
Debug.Assert(tokens [7].Content == "and");
Debug.Assert(tokens [7].Desc == "AND");
Debug.Assert(tokens [11].Content == "23232");
Debug.Assert(tokens [11].Desc == "STRING");
tokens = lex.Tokenize("//label[@text=\"ceva\" and @tre=\"23232\" and @rez=\"ultim and\"]");
Debug.Assert(tokens.Count == 19);
Debug.Assert(tokens [3].Content == "text");
Debug.Assert(tokens [3].Desc == "ATTRIBUTE");
Debug.Assert(tokens [4].Content == "=");
Debug.Assert(tokens [4].Desc == "EQUAL");
Debug.Assert(tokens [5].Content == "ceva");
Debug.Assert(tokens [5].Desc == "STRING");
Debug.Assert(tokens [7].Content == "and");
Debug.Assert(tokens [7].Desc == "AND");
Debug.Assert(tokens [11].Content == "23232");
Debug.Assert(tokens [11].Desc == "STRING");
Debug.Assert(tokens [13].Content == "and");
Debug.Assert(tokens [13].Desc == "AND");
Debug.Assert(tokens [15].Content == "rez");
Debug.Assert(tokens [15].Desc == "ATTRIBUTE");
Debug.Assert(tokens [17].Content == "ultim and");
Debug.Assert(tokens [17].Desc == "STRING");
}
public List <XPathNode> Parse(string xPath)
{
List <XPathNode> ret = new List <XPathNode> ();
//adding // if the path doesn't
if (xPath.StartsWith("/") == false && xPath.StartsWith("//") == false)
{
xPath = "//" + xPath;
}
List <Token> tokens = _lexer.Tokenize(xPath);
int currentIndex = 0;
while (currentIndex < tokens.Count)
{
//searching for ancestry
XPathNode newNode = new XPathNode();
newNode.IsChild = true;
if (tokens [currentIndex].Desc == "ANCESTOR")
{
newNode.IsChild = false;
}
currentIndex++;
while (tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
newNode.TagName = tokens [currentIndex].Content;
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
//check for attributes
if (currentIndex < tokens.Count && tokens [currentIndex].Desc == "OPEN_PREDICATE")
{
//maybe the input is broken and we don't have
//a closing bracket
while (currentIndex < tokens.Count && tokens [currentIndex].Desc != "END_PREDICATE")
{
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
//number
if (tokens [currentIndex].Desc == "NUMBER")
{
var numberPredicate = new XPathNumberPredicate();
numberPredicate.Number = int.Parse(tokens[currentIndex].Content);
newNode.predicates.Add(numberPredicate);
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
//we have some junk in here
//we gonna parse until here and bail
if (tokens [currentIndex].Desc != "END_PREDICATE")
{
ret.Add(newNode);
break;
}
}
//attribute
if (tokens [currentIndex].Desc == "ATTRIBUTE")
{
XPathAttribute attribute = new XPathAttribute();
attribute.Name = tokens [currentIndex].Content;
newNode.predicates.Add(attribute);
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
//maybe we have a test for existence of attribute
if (tokens [currentIndex].Desc != "END_PREDICATE")
{
//how to compare
if (tokens [currentIndex].Desc == "EQUAL")
{
}
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
attribute.ValueToMatch = tokens [currentIndex].Content;
currentIndex++;
while (currentIndex < tokens.Count && tokens [currentIndex].Desc == "WHITE_SPACE")
{
currentIndex++;
}
if (currentIndex < tokens.Count && tokens [currentIndex].Desc == "AND")
{
currentIndex++;
}
}
}
}
currentIndex++;
}
ret.Add(newNode);
}
return(ret);
}
/// <summary>
/// コンパイルするコードを指定してコンパイルし、実行します。
/// </summary>
/// <param name="code">コンパイルされるコード文字列。</param>
/// <returns>コンパイルに成功したとき true、それ以外のとき false。</returns>
public LuryObject Evaluate(string code)
{
var lexer = new Lexer.Lexer(code + '\n');
var succeedTokenize = lexer.Tokenize();
lexer.Logger.CopyTo(this.OutputLogger);
if (!succeedTokenize)
return null;
var globalContext = new LuryContext();
Intrinsic.SetBuiltInFunctions(globalContext);
var parser = new Parser();
Routine routine = null;
// parsing
try
{
routine = (Routine)parser.yyparse(new Lex2yyInput(lexer), new yydebug.yyDebugSimple());
}
catch (yyParser.yyException ex)
{
this.ReportyyException(ex, code);
return null;
}
// running
if (routine == null)
return null;
else
{
try
{
var exit = routine.Evaluate(globalContext);
if (exit.ExitReason == StatementExitReason.Break)
throw new LuryException(LuryExceptionType.WrongBreak);
return exit.ReturnValue;
}
catch (LuryException ex)
{
Console.Error.WriteLine(ex.Message);
return null;
}
}
}
