public void verify_analysis_apathy_to_culture_settings()
{
var current = Thread.CurrentThread.CurrentCulture;
var lexer = new Lexer();
foreach (var temp in CultureInfo.GetCultures(CultureTypes.AllCultures))
{
Thread.CurrentThread.CurrentCulture = temp;
// literals parsing should not vary across diverse cultures
var tkn = lexer.Analyze("0.1").First(); // e.g. double literal should be always written using dot in our expressions language, no matter the culture
Assert.Equal(TokenType.FLOAT, tkn.Type);
Assert.Equal(0.1, tkn.Value);
}
Thread.CurrentThread.CurrentCulture = current;
}
/// <summary>
/// Parse the text and build error and token lists.
/// </summary>
private void Parse()
{
lexer = new Lexer(buffer.CurrentSnapshot.GetText());
lexer.Analyze();
tagList.Clear();
// copy the lexer.Token list to an array, because it can be changed by the parser in the background
// this should not happen, but under some circumstandings it happens (and I don't know when and why)
foreach (var message in lexer.Errors.ToArray())
{
// check the length of the new span, should not be longer than the current text
var length = message.Length;
if (message.Position + message.Length > buffer.CurrentSnapshot.Length)
{
length = buffer.CurrentSnapshot.Length - message.Position;
}
if (length > 0)
{
var newSpan = new Span(message.Position, length);
tagList.Add(new TagSpan <ProtobufTokenTag>(new SnapshotSpan(buffer.CurrentSnapshot, newSpan), new ProtobufErrorTag(message.Message)));
}
}
// copy the lexer.Token list to an array, because it can be changed by the parser in the background
// this should not happen, but under some circumstandings it happens (and I don't know when and why)
foreach (var token in lexer.Tokens.ToArray())
{
// check the length of the new span, should not be longer than the current text
var length = token.Length;
if (token.Position + token.Length > buffer.CurrentSnapshot.Length)
{
length = buffer.CurrentSnapshot.Length - token.Position;
}
if (length > 0)
{
var newSpan = new Span(token.Position, length);
tagList.Add(new TagSpan <ProtobufTokenTag>(new SnapshotSpan(buffer.CurrentSnapshot, newSpan), new ProtobufTokenTag(token.CodeType)));
}
}
}
public void verify_invalid_tokens_detection()
{
var lexer = new Lexer();
try
{
lexer.Analyze(null);
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ArgumentNullException);
Assert.AreEqual(e.Message, "Expression not provided.\r\nParameter name: expression");
}
try
{
lexer.Analyze("true # false");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ParseErrorException);
Assert.AreEqual(e.Message, "Invalid token.");
var ctx = ((ParseErrorException)e).Context;
Assert.AreEqual(ctx.Expression, "# false");
Assert.AreEqual(ctx.Line, 1);
Assert.AreEqual(ctx.Column, 6);
}
try
{
lexer.Analyze("true && ^");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ParseErrorException);
Assert.AreEqual(e.Message, "Invalid token.");
var ctx = ((ParseErrorException)e).Context;
Assert.AreEqual(ctx.Expression, "^");
Assert.AreEqual(ctx.Line, 1);
Assert.AreEqual(ctx.Column, 9);
}
try
{
lexer.Analyze("'John's cat'");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ParseErrorException);
Assert.AreEqual(e.Message, "Invalid token.");
var ctx = ((ParseErrorException)e).Context;
Assert.AreEqual(ctx.Expression, "'");
Assert.AreEqual(ctx.Line, 1);
Assert.AreEqual(ctx.Column, 12);
}
}
public void verify_complex_expression_analysis()
{
const string expression =
@"GoAbroad == true
&& (
(NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0)
|| (Age > 24 && Age <= 55.5)
&& (1+2*2>1-2/2)
)";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression).ToArray();
Assert.AreEqual(tokens.Length, 45);
Assert.AreEqual(tokens[0].Value, "GoAbroad");
Assert.AreEqual(tokens[0].Type, TokenType.FUNC);
Assert.AreEqual(tokens[1].Value, "==");
Assert.AreEqual(tokens[1].Type, TokenType.EQ);
Assert.AreEqual(tokens[2].Value, true);
Assert.AreEqual(tokens[2].Type, TokenType.BOOL);
Assert.AreEqual(tokens[3].Value, "&&");
Assert.AreEqual(tokens[3].Type, TokenType.AND);
Assert.AreEqual(tokens[4].Value, "(");
Assert.AreEqual(tokens[4].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[5].Value, "(");
Assert.AreEqual(tokens[5].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[6].Value, "NextCountry");
Assert.AreEqual(tokens[6].Type, TokenType.FUNC);
Assert.AreEqual(tokens[7].Value, "!=");
Assert.AreEqual(tokens[7].Type, TokenType.NEQ);
Assert.AreEqual(tokens[8].Value, "european country");
Assert.AreEqual(tokens[8].Type, TokenType.STRING);
Assert.AreEqual(tokens[9].Value, "&&");
Assert.AreEqual(tokens[9].Type, TokenType.AND);
Assert.AreEqual(tokens[10].Value, "Compare");
Assert.AreEqual(tokens[10].Type, TokenType.FUNC);
Assert.AreEqual(tokens[11].Value, "(");
Assert.AreEqual(tokens[11].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[12].Value, "NextCountry");
Assert.AreEqual(tokens[12].Type, TokenType.FUNC);
Assert.AreEqual(tokens[13].Value, ",");
Assert.AreEqual(tokens[13].Type, TokenType.COMMA);
Assert.AreEqual(tokens[14].Value, "Country.Name");
Assert.AreEqual(tokens[14].Type, TokenType.FUNC);
Assert.AreEqual(tokens[15].Value, ")");
Assert.AreEqual(tokens[15].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[16].Value, "==");
Assert.AreEqual(tokens[16].Type, TokenType.EQ);
Assert.AreEqual(tokens[17].Value, 0);
Assert.AreEqual(tokens[17].Type, TokenType.INT);
Assert.AreEqual(tokens[18].Value, ")");
Assert.AreEqual(tokens[18].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[19].Value, "||");
Assert.AreEqual(tokens[19].Type, TokenType.OR);
Assert.AreEqual(tokens[20].Value, "(");
Assert.AreEqual(tokens[20].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[21].Value, "Age");
Assert.AreEqual(tokens[21].Type, TokenType.FUNC);
Assert.AreEqual(tokens[22].Value, ">");
Assert.AreEqual(tokens[22].Type, TokenType.GT);
Assert.AreEqual(tokens[23].Value, 24);
Assert.AreEqual(tokens[23].Type, TokenType.INT);
Assert.AreEqual(tokens[24].Value, "&&");
Assert.AreEqual(tokens[24].Type, TokenType.AND);
Assert.AreEqual(tokens[25].Value, "Age");
Assert.AreEqual(tokens[25].Type, TokenType.FUNC);
Assert.AreEqual(tokens[26].Value, "<=");
Assert.AreEqual(tokens[26].Type, TokenType.LE);
Assert.AreEqual(tokens[27].Value, 55.5d);
Assert.AreEqual(tokens[27].Type, TokenType.FLOAT);
Assert.AreEqual(tokens[28].Value, ")");
Assert.AreEqual(tokens[28].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[29].Value, "&&");
Assert.AreEqual(tokens[29].Type, TokenType.AND);
Assert.AreEqual(tokens[30].Value, "(");
Assert.AreEqual(tokens[30].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[31].Value, 1);
Assert.AreEqual(tokens[31].Type, TokenType.INT);
Assert.AreEqual(tokens[32].Value, "+");
Assert.AreEqual(tokens[32].Type, TokenType.ADD);
Assert.AreEqual(tokens[33].Value, 2);
Assert.AreEqual(tokens[33].Type, TokenType.INT);
Assert.AreEqual(tokens[34].Value, "*");
Assert.AreEqual(tokens[34].Type, TokenType.MUL);
Assert.AreEqual(tokens[35].Value, 2);
Assert.AreEqual(tokens[35].Type, TokenType.INT);
Assert.AreEqual(tokens[36].Value, ">");
Assert.AreEqual(tokens[36].Type, TokenType.GT);
Assert.AreEqual(tokens[37].Value, 1);
Assert.AreEqual(tokens[37].Type, TokenType.INT);
Assert.AreEqual(tokens[38].Value, "-");
Assert.AreEqual(tokens[38].Type, TokenType.SUB);
Assert.AreEqual(tokens[39].Value, 2);
Assert.AreEqual(tokens[39].Type, TokenType.INT);
Assert.AreEqual(tokens[40].Value, "/");
Assert.AreEqual(tokens[40].Type, TokenType.DIV);
Assert.AreEqual(tokens[41].Value, 2);
Assert.AreEqual(tokens[41].Type, TokenType.INT);
Assert.AreEqual(tokens[42].Value, ")");
Assert.AreEqual(tokens[42].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[43].Value, ")");
Assert.AreEqual(tokens[43].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[44].Value, string.Empty);
Assert.AreEqual(tokens[44].Type, TokenType.EOF);
}
public void verify_complex_expression_analysis()
{
const string expression =
@"GoAbroad == true
&& (
(NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0)
|| (Age > 24 && Age <= 55.5)
&& (1+2*2>1-2/2)
)";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression).ToArray();
Assert.AreEqual(45, tokens.Length);
Assert.AreEqual("GoAbroad", tokens[0].Value);
Assert.AreEqual(TokenType.FUNC, tokens[0].Type);
Assert.AreEqual("==", tokens[1].Value);
Assert.AreEqual(TokenType.EQ, tokens[1].Type);
Assert.AreEqual(true, tokens[2].Value);
Assert.AreEqual(TokenType.BOOL, tokens[2].Type);
Assert.AreEqual("&&", tokens[3].Value);
Assert.AreEqual(TokenType.AND, tokens[3].Type);
Assert.AreEqual("(", tokens[4].Value);
Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[4].Type);
Assert.AreEqual("(", tokens[5].Value);
Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[5].Type);
Assert.AreEqual("NextCountry", tokens[6].Value);
Assert.AreEqual(TokenType.FUNC, tokens[6].Type);
Assert.AreEqual("!=", tokens[7].Value);
Assert.AreEqual(TokenType.NEQ, tokens[7].Type);
Assert.AreEqual("european country", tokens[8].Value);
Assert.AreEqual(TokenType.STRING, tokens[8].Type);
Assert.AreEqual("&&", tokens[9].Value);
Assert.AreEqual(TokenType.AND, tokens[9].Type);
Assert.AreEqual("Compare", tokens[10].Value);
Assert.AreEqual(TokenType.FUNC, tokens[10].Type);
Assert.AreEqual("(", tokens[11].Value);
Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[11].Type);
Assert.AreEqual("NextCountry", tokens[12].Value);
Assert.AreEqual(TokenType.FUNC, tokens[12].Type);
Assert.AreEqual(",", tokens[13].Value);
Assert.AreEqual(TokenType.COMMA, tokens[13].Type);
Assert.AreEqual("Country.Name", tokens[14].Value);
Assert.AreEqual(TokenType.FUNC, tokens[14].Type);
Assert.AreEqual(")", tokens[15].Value);
Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[15].Type);
Assert.AreEqual("==", tokens[16].Value);
Assert.AreEqual(TokenType.EQ, tokens[16].Type);
Assert.AreEqual(0, tokens[17].Value);
Assert.AreEqual(TokenType.INT, tokens[17].Type);
Assert.AreEqual(")", tokens[18].Value);
Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[18].Type);
Assert.AreEqual("||", tokens[19].Value);
Assert.AreEqual(TokenType.OR, tokens[19].Type);
Assert.AreEqual("(", tokens[20].Value);
Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[20].Type);
Assert.AreEqual("Age", tokens[21].Value);
Assert.AreEqual(TokenType.FUNC, tokens[21].Type);
Assert.AreEqual(">", tokens[22].Value);
Assert.AreEqual(TokenType.GT, tokens[22].Type);
Assert.AreEqual(24, tokens[23].Value);
Assert.AreEqual(TokenType.INT, tokens[23].Type);
Assert.AreEqual("&&", tokens[24].Value);
Assert.AreEqual(TokenType.AND, tokens[24].Type);
Assert.AreEqual("Age", tokens[25].Value);
Assert.AreEqual(TokenType.FUNC, tokens[25].Type);
Assert.AreEqual("<=", tokens[26].Value);
Assert.AreEqual(TokenType.LE, tokens[26].Type);
Assert.AreEqual(55.5d, tokens[27].Value);
Assert.AreEqual(TokenType.FLOAT, tokens[27].Type);
Assert.AreEqual(")", tokens[28].Value);
Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[28].Type);
Assert.AreEqual("&&", tokens[29].Value);
Assert.AreEqual(TokenType.AND, tokens[29].Type);
Assert.AreEqual("(", tokens[30].Value);
Assert.AreEqual(TokenType.LEFT_BRACKET, tokens[30].Type);
Assert.AreEqual(1, tokens[31].Value);
Assert.AreEqual(TokenType.INT, tokens[31].Type);
Assert.AreEqual("+", tokens[32].Value);
Assert.AreEqual(TokenType.ADD, tokens[32].Type);
Assert.AreEqual(2, tokens[33].Value);
Assert.AreEqual(TokenType.INT, tokens[33].Type);
Assert.AreEqual("*", tokens[34].Value);
Assert.AreEqual(TokenType.MUL, tokens[34].Type);
Assert.AreEqual(2, tokens[35].Value);
Assert.AreEqual(TokenType.INT, tokens[35].Type);
Assert.AreEqual(">", tokens[36].Value);
Assert.AreEqual(TokenType.GT, tokens[36].Type);
Assert.AreEqual(1, tokens[37].Value);
Assert.AreEqual(TokenType.INT, tokens[37].Type);
Assert.AreEqual("-", tokens[38].Value);
Assert.AreEqual(TokenType.SUB, tokens[38].Type);
Assert.AreEqual(2, tokens[39].Value);
Assert.AreEqual(TokenType.INT, tokens[39].Type);
Assert.AreEqual("/", tokens[40].Value);
Assert.AreEqual(TokenType.DIV, tokens[40].Type);
Assert.AreEqual(2, tokens[41].Value);
Assert.AreEqual(TokenType.INT, tokens[41].Type);
Assert.AreEqual(")", tokens[42].Value);
Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[42].Type);
Assert.AreEqual(")", tokens[43].Value);
Assert.AreEqual(TokenType.RIGHT_BRACKET, tokens[43].Type);
Assert.AreEqual(string.Empty, tokens[44].Value);
Assert.AreEqual(TokenType.EOF, tokens[44].Type);
}
public void Parse_InputInvalidUnaryOperator_Exception(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.Catch <Exception>(() => Parser.Parse(tokens));
}
public void verify_complex_expression_analysis()
{
const string expression =
@"GoAbroad == true
&& (
(NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0)
|| (Age > 24 && Age <= 55.5)
&&(1.1+2*2>1-2/2+Array[0].Value=='\'\na\n b\nc\n\'')
)";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression).ToArray();
Assert.Equal(49, tokens.Length);
Assert.Equal("GoAbroad", tokens[0].Value);
Assert.Equal(TokenType.FUNC, tokens[0].Type);
Assert.Equal("==", tokens[1].Value);
Assert.Equal(TokenType.EQ, tokens[1].Type);
Assert.Equal(true, tokens[2].Value);
Assert.Equal(TokenType.BOOL, tokens[2].Type);
Assert.Equal("&&", tokens[3].Value);
Assert.Equal(TokenType.AND, tokens[3].Type);
Assert.Equal("(", tokens[4].Value);
Assert.Equal(TokenType.LEFT_BRACKET, tokens[4].Type);
Assert.Equal("(", tokens[5].Value);
Assert.Equal(TokenType.LEFT_BRACKET, tokens[5].Type);
Assert.Equal("NextCountry", tokens[6].Value);
Assert.Equal(TokenType.FUNC, tokens[6].Type);
Assert.Equal("!=", tokens[7].Value);
Assert.Equal(TokenType.NEQ, tokens[7].Type);
Assert.Equal("european country", tokens[8].Value);
Assert.Equal(TokenType.STRING, tokens[8].Type);
Assert.Equal("&&", tokens[9].Value);
Assert.Equal(TokenType.AND, tokens[9].Type);
Assert.Equal("Compare", tokens[10].Value);
Assert.Equal(TokenType.FUNC, tokens[10].Type);
Assert.Equal("(", tokens[11].Value);
Assert.Equal(TokenType.LEFT_BRACKET, tokens[11].Type);
Assert.Equal("NextCountry", tokens[12].Value);
Assert.Equal(TokenType.FUNC, tokens[12].Type);
Assert.Equal(",", tokens[13].Value);
Assert.Equal(TokenType.COMMA, tokens[13].Type);
Assert.Equal("Country.Name", tokens[14].Value);
Assert.Equal(TokenType.FUNC, tokens[14].Type);
Assert.Equal(")", tokens[15].Value);
Assert.Equal(TokenType.RIGHT_BRACKET, tokens[15].Type);
Assert.Equal("==", tokens[16].Value);
Assert.Equal(TokenType.EQ, tokens[16].Type);
Assert.Equal(0, tokens[17].Value);
Assert.Equal(TokenType.INT, tokens[17].Type);
Assert.Equal(")", tokens[18].Value);
Assert.Equal(TokenType.RIGHT_BRACKET, tokens[18].Type);
Assert.Equal("||", tokens[19].Value);
Assert.Equal(TokenType.OR, tokens[19].Type);
Assert.Equal("(", tokens[20].Value);
Assert.Equal(TokenType.LEFT_BRACKET, tokens[20].Type);
Assert.Equal("Age", tokens[21].Value);
Assert.Equal(TokenType.FUNC, tokens[21].Type);
Assert.Equal(">", tokens[22].Value);
Assert.Equal(TokenType.GT, tokens[22].Type);
Assert.Equal(24, tokens[23].Value);
Assert.Equal(TokenType.INT, tokens[23].Type);
Assert.Equal("&&", tokens[24].Value);
Assert.Equal(TokenType.AND, tokens[24].Type);
Assert.Equal("Age", tokens[25].Value);
Assert.Equal(TokenType.FUNC, tokens[25].Type);
Assert.Equal("<=", tokens[26].Value);
Assert.Equal(TokenType.LE, tokens[26].Type);
Assert.Equal(55.5d, tokens[27].Value);
Assert.Equal(TokenType.FLOAT, tokens[27].Type);
Assert.Equal(")", tokens[28].Value);
Assert.Equal(TokenType.RIGHT_BRACKET, tokens[28].Type);
Assert.Equal("&&", tokens[29].Value);
Assert.Equal(TokenType.AND, tokens[29].Type);
Assert.Equal("(", tokens[30].Value);
Assert.Equal(TokenType.LEFT_BRACKET, tokens[30].Type);
Assert.Equal(1.1, tokens[31].Value);
Assert.Equal(TokenType.FLOAT, tokens[31].Type);
Assert.Equal("+", tokens[32].Value);
Assert.Equal(TokenType.ADD, tokens[32].Type);
Assert.Equal(2, tokens[33].Value);
Assert.Equal(TokenType.INT, tokens[33].Type);
Assert.Equal("*", tokens[34].Value);
Assert.Equal(TokenType.MUL, tokens[34].Type);
Assert.Equal(2, tokens[35].Value);
Assert.Equal(TokenType.INT, tokens[35].Type);
Assert.Equal(">", tokens[36].Value);
Assert.Equal(TokenType.GT, tokens[36].Type);
Assert.Equal(1, tokens[37].Value);
Assert.Equal(TokenType.INT, tokens[37].Type);
Assert.Equal("-", tokens[38].Value);
Assert.Equal(TokenType.SUB, tokens[38].Type);
Assert.Equal(2, tokens[39].Value);
Assert.Equal(TokenType.INT, tokens[39].Type);
Assert.Equal("/", tokens[40].Value);
Assert.Equal(TokenType.DIV, tokens[40].Type);
Assert.Equal(2, tokens[41].Value);
Assert.Equal(TokenType.INT, tokens[41].Type);
Assert.Equal("+", tokens[42].Value);
Assert.Equal(TokenType.ADD, tokens[42].Type);
Assert.Equal("Array[0].Value", tokens[43].Value);
Assert.Equal(TokenType.FUNC, tokens[43].Type);
Assert.Equal("==", tokens[44].Value);
Assert.Equal(TokenType.EQ, tokens[44].Type);
Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[45].Value); // used alternatively to verbatim string (new line \n in expression has been replaced by windows \r\n)
Assert.Equal(TokenType.STRING, tokens[45].Type);
Assert.Equal(")", tokens[46].Value);
Assert.Equal(TokenType.RIGHT_BRACKET, tokens[46].Type);
Assert.Equal(")", tokens[47].Value);
Assert.Equal(TokenType.RIGHT_BRACKET, tokens[47].Type);
Assert.Equal(string.Empty, tokens[48].Value);
Assert.Equal(TokenType.EOF, tokens[48].Type);
}
public void Verify_lexer_logic()
{
const string expression =
"GoAbroad == true " +
"&& (" +
"(NextCountry != \"european country\" && NextCountry == Country.Name) " +
"|| (Age > 24 && Age <= 55.5)" +
")";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression);
Assert.AreEqual(tokens.Length, 25);
Assert.AreEqual(tokens[0].Value, "GoAbroad");
Assert.AreEqual(tokens[1].Value, "==");
Assert.AreEqual(tokens[2].Value, true);
Assert.AreEqual(tokens[3].Value, "&&");
Assert.AreEqual(tokens[4].Value, "(");
Assert.AreEqual(tokens[5].Value, "(");
Assert.AreEqual(tokens[6].Value, "NextCountry");
Assert.AreEqual(tokens[7].Value, "!=");
Assert.AreEqual(tokens[8].Value, "european country");
Assert.AreEqual(tokens[9].Value, "&&");
Assert.AreEqual(tokens[10].Value, "NextCountry");
Assert.AreEqual(tokens[11].Value, "==");
Assert.AreEqual(tokens[12].Value, "Country.Name");
Assert.AreEqual(tokens[13].Value, ")");
Assert.AreEqual(tokens[14].Value, "||");
Assert.AreEqual(tokens[15].Value, "(");
Assert.AreEqual(tokens[16].Value, "Age");
Assert.AreEqual(tokens[17].Value, ">");
Assert.AreEqual(tokens[18].Value, 24);
Assert.AreEqual(tokens[19].Value, "&&");
Assert.AreEqual(tokens[20].Value, "Age");
Assert.AreEqual(tokens[21].Value, "<=");
Assert.AreEqual(tokens[22].Value, 55.5f);
Assert.AreEqual(tokens[23].Value, ")");
Assert.AreEqual(tokens[24].Value, ")");
try
{
lexer.Analyze("true + false");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ArgumentException);
Assert.IsTrue(e.Message == "Lexer error. Unexpected token started at + false.");
}
try
{
lexer.Analyze("true && ^");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ArgumentException);
Assert.IsTrue(e.Message == "Lexer error. Unexpected token started at ^.");
}
}
public void Analyze_InputUnclosedString_Exception(string text)
{
Assert.Catch <Exception>(() => Lexer.Analyze(text).ToList());
}
public void Analyze_InputIncompleteOperator_Exception(string text)
{
Assert.Catch <Exception>(() => Lexer.Analyze(text).ToList());
}
public void Analyze_InputLexeme_ReturnOne(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.AreEqual(1, tokens.Count);
}
public void Analyze_InputInvalidChar_Exception()
{
const string text = "♥";
Assert.Catch <Exception>(() => Lexer.Analyze(text).ToList());
}
public void Analyze_InputNumberNoFractionalPart_Exception(string text)
{
Assert.Catch <Exception>(() => Lexer.Analyze(text).ToList());
}
public void Parse_InputIncompleteBlock_Exception(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.Catch <Exception>(() => Parser.Parse(tokens));
}
public void Analyze_InputKeyword_ReturnKeyword(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.AreEqual(TokenType.Keyword, tokens[0].Type);
}
private void InitTokenizer(string expression)
{
var lexer = new Lexer();
Tokens = new Stack <TokenOutput>(lexer.Analyze(expression).Reverse());
}
public void Analyze_InputWord_ReturnLiteral(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.AreEqual(TokenType.Literal, tokens[0].Type);
}
public void verify_valid_tokens_extraction()
{
const string expression =
@"! || && == != < <= > >= + - * / % ~ & ^ | << >> () [] . ? :
null true false 123 0.3e-2 0b1010 0xFF '\'\na\n b\nc\n\'' メidメ";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression).ToArray();
Assert.Equal(37, tokens.Length);
Assert.Equal("!", tokens[0].Value);
Assert.Equal(TokenType.L_NOT, tokens[0].Type);
Assert.Equal("||", tokens[1].Value);
Assert.Equal(TokenType.L_OR, tokens[1].Type);
Assert.Equal("&&", tokens[2].Value);
Assert.Equal(TokenType.L_AND, tokens[2].Type);
Assert.Equal("==", tokens[3].Value);
Assert.Equal(TokenType.EQ, tokens[3].Type);
Assert.Equal("!=", tokens[4].Value);
Assert.Equal(TokenType.NEQ, tokens[4].Type);
Assert.Equal("<", tokens[5].Value);
Assert.Equal(TokenType.LT, tokens[5].Type);
Assert.Equal("<=", tokens[6].Value);
Assert.Equal(TokenType.LE, tokens[6].Type);
Assert.Equal(">", tokens[7].Value);
Assert.Equal(TokenType.GT, tokens[7].Type);
Assert.Equal(">=", tokens[8].Value);
Assert.Equal(TokenType.GE, tokens[8].Type);
Assert.Equal("+", tokens[9].Value);
Assert.Equal(TokenType.ADD, tokens[9].Type);
Assert.Equal("-", tokens[10].Value);
Assert.Equal(TokenType.SUB, tokens[10].Type);
Assert.Equal("*", tokens[11].Value);
Assert.Equal(TokenType.MUL, tokens[11].Type);
Assert.Equal("/", tokens[12].Value);
Assert.Equal(TokenType.DIV, tokens[12].Type);
Assert.Equal("%", tokens[13].Value);
Assert.Equal(TokenType.MOD, tokens[13].Type);
Assert.Equal("~", tokens[14].Value);
Assert.Equal(TokenType.B_NOT, tokens[14].Type);
Assert.Equal("&", tokens[15].Value);
Assert.Equal(TokenType.B_AND, tokens[15].Type);
Assert.Equal("^", tokens[16].Value);
Assert.Equal(TokenType.XOR, tokens[16].Type);
Assert.Equal("|", tokens[17].Value);
Assert.Equal(TokenType.B_OR, tokens[17].Type);
Assert.Equal("<<", tokens[18].Value);
Assert.Equal(TokenType.L_SHIFT, tokens[18].Type);
Assert.Equal(">>", tokens[19].Value);
Assert.Equal(TokenType.R_SHIFT, tokens[19].Type);
Assert.Equal("(", tokens[20].Value);
Assert.Equal(TokenType.L_PAR, tokens[20].Type);
Assert.Equal(")", tokens[21].Value);
Assert.Equal(TokenType.R_PAR, tokens[21].Type);
Assert.Equal("[", tokens[22].Value);
Assert.Equal(TokenType.L_BRACKET, tokens[22].Type);
Assert.Equal("]", tokens[23].Value);
Assert.Equal(TokenType.R_BRACKET, tokens[23].Type);
Assert.Equal(".", tokens[24].Value);
Assert.Equal(TokenType.PERIOD, tokens[24].Type);
Assert.Equal("?", tokens[25].Value);
Assert.Equal(TokenType.QMARK, tokens[25].Type);
Assert.Equal(":", tokens[26].Value);
Assert.Equal(TokenType.COLON, tokens[26].Type);
Assert.Equal(null, tokens[27].Value);
Assert.Equal(TokenType.NULL, tokens[27].Type);
Assert.Equal(true, tokens[28].Value);
Assert.Equal(TokenType.BOOL, tokens[28].Type);
Assert.Equal(false, tokens[29].Value);
Assert.Equal(TokenType.BOOL, tokens[29].Type);
Assert.Equal(123, tokens[30].Value);
Assert.Equal(TokenType.INT, tokens[30].Type);
Assert.Equal(0.3e-2, tokens[31].Value);
Assert.Equal(TokenType.FLOAT, tokens[31].Type);
Assert.Equal(10, tokens[32].Value);
Assert.Equal(TokenType.BIN, tokens[32].Type);
Assert.Equal(255, tokens[33].Value);
Assert.Equal(TokenType.HEX, tokens[33].Type);
Assert.Equal("'\r\na\r\n b\r\nc\r\n'", tokens[34].Value); // used alternatively to verbatim string (new line \n in expression string literal has been replaced by windows \r\n)
Assert.Equal(TokenType.STRING, tokens[34].Type);
Assert.Equal("メidメ", tokens[35].Value);
Assert.Equal(TokenType.ID, tokens[35].Type);
Assert.Equal(string.Empty, tokens[36].Value);
Assert.Equal(TokenType.EOF, tokens[36].Type);
}
public void Analyze_InputWord_ReturnIdentifier(string text)
{
var tokens = Lexer.Analyze(text).ToList();
Assert.AreEqual(TokenType.Identifier, tokens[0].Type);
}
public void verify_complex_expression_analysis()
{
const string expression =
"GoAbroad == true " +
"&& (" +
"(NextCountry != 'european country' && Compare(NextCountry, Country.Name) == 0) " +
"|| (Age > 24 && Age <= 55.5)" +
"&& (1 + 2 * 2 > 1 - 2 / 2)" +
")";
var lexer = new Lexer();
var tokens = lexer.Analyze(expression).ToArray();
Assert.AreEqual(tokens.Length, 45);
Assert.AreEqual(tokens[0].Value, "GoAbroad");
Assert.AreEqual(tokens[0].Type, TokenType.FUNC);
Assert.AreEqual(tokens[1].Value, "==");
Assert.AreEqual(tokens[1].Type, TokenType.EQ);
Assert.AreEqual(tokens[2].Value, true);
Assert.AreEqual(tokens[2].Type, TokenType.BOOL);
Assert.AreEqual(tokens[3].Value, "&&");
Assert.AreEqual(tokens[3].Type, TokenType.AND);
Assert.AreEqual(tokens[4].Value, "(");
Assert.AreEqual(tokens[4].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[5].Value, "(");
Assert.AreEqual(tokens[5].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[6].Value, "NextCountry");
Assert.AreEqual(tokens[6].Type, TokenType.FUNC);
Assert.AreEqual(tokens[7].Value, "!=");
Assert.AreEqual(tokens[7].Type, TokenType.NEQ);
Assert.AreEqual(tokens[8].Value, "european country");
Assert.AreEqual(tokens[8].Type, TokenType.STRING);
Assert.AreEqual(tokens[9].Value, "&&");
Assert.AreEqual(tokens[9].Type, TokenType.AND);
Assert.AreEqual(tokens[10].Value, "Compare");
Assert.AreEqual(tokens[10].Type, TokenType.FUNC);
Assert.AreEqual(tokens[11].Value, "(");
Assert.AreEqual(tokens[11].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[12].Value, "NextCountry");
Assert.AreEqual(tokens[12].Type, TokenType.FUNC);
Assert.AreEqual(tokens[13].Value, ",");
Assert.AreEqual(tokens[13].Type, TokenType.COMMA);
Assert.AreEqual(tokens[14].Value, "Country.Name");
Assert.AreEqual(tokens[14].Type, TokenType.FUNC);
Assert.AreEqual(tokens[15].Value, ")");
Assert.AreEqual(tokens[15].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[16].Value, "==");
Assert.AreEqual(tokens[16].Type, TokenType.EQ);
Assert.AreEqual(tokens[17].Value, 0);
Assert.AreEqual(tokens[17].Type, TokenType.INT);
Assert.AreEqual(tokens[18].Value, ")");
Assert.AreEqual(tokens[18].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[19].Value, "||");
Assert.AreEqual(tokens[19].Type, TokenType.OR);
Assert.AreEqual(tokens[20].Value, "(");
Assert.AreEqual(tokens[20].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[21].Value, "Age");
Assert.AreEqual(tokens[21].Type, TokenType.FUNC);
Assert.AreEqual(tokens[22].Value, ">");
Assert.AreEqual(tokens[22].Type, TokenType.GT);
Assert.AreEqual(tokens[23].Value, 24);
Assert.AreEqual(tokens[23].Type, TokenType.INT);
Assert.AreEqual(tokens[24].Value, "&&");
Assert.AreEqual(tokens[24].Type, TokenType.AND);
Assert.AreEqual(tokens[25].Value, "Age");
Assert.AreEqual(tokens[25].Type, TokenType.FUNC);
Assert.AreEqual(tokens[26].Value, "<=");
Assert.AreEqual(tokens[26].Type, TokenType.LE);
Assert.AreEqual(tokens[27].Value, 55.5d);
Assert.AreEqual(tokens[27].Type, TokenType.FLOAT);
Assert.AreEqual(tokens[28].Value, ")");
Assert.AreEqual(tokens[28].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[29].Value, "&&");
Assert.AreEqual(tokens[29].Type, TokenType.AND);
Assert.AreEqual(tokens[30].Value, "(");
Assert.AreEqual(tokens[30].Type, TokenType.LEFT_BRACKET);
Assert.AreEqual(tokens[31].Value, 1);
Assert.AreEqual(tokens[31].Type, TokenType.INT);
Assert.AreEqual(tokens[32].Value, "+");
Assert.AreEqual(tokens[32].Type, TokenType.ADD);
Assert.AreEqual(tokens[33].Value, 2);
Assert.AreEqual(tokens[33].Type, TokenType.INT);
Assert.AreEqual(tokens[34].Value, "*");
Assert.AreEqual(tokens[34].Type, TokenType.MUL);
Assert.AreEqual(tokens[35].Value, 2);
Assert.AreEqual(tokens[35].Type, TokenType.INT);
Assert.AreEqual(tokens[36].Value, ">");
Assert.AreEqual(tokens[36].Type, TokenType.GT);
Assert.AreEqual(tokens[37].Value, 1);
Assert.AreEqual(tokens[37].Type, TokenType.INT);
Assert.AreEqual(tokens[38].Value, "-");
Assert.AreEqual(tokens[38].Type, TokenType.SUB);
Assert.AreEqual(tokens[39].Value, 2);
Assert.AreEqual(tokens[39].Type, TokenType.INT);
Assert.AreEqual(tokens[40].Value, "/");
Assert.AreEqual(tokens[40].Type, TokenType.DIV);
Assert.AreEqual(tokens[41].Value, 2);
Assert.AreEqual(tokens[41].Type, TokenType.INT);
Assert.AreEqual(tokens[42].Value, ")");
Assert.AreEqual(tokens[42].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[43].Value, ")");
Assert.AreEqual(tokens[43].Type, TokenType.RIGHT_BRACKET);
Assert.AreEqual(tokens[44].Value, string.Empty);
Assert.AreEqual(tokens[44].Type, TokenType.EOF);
try
{
lexer.Analyze(null);
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is ArgumentNullException);
Assert.IsTrue(e.Message == "Expression not provided.\r\nParameter name: expression");
}
try
{
lexer.Analyze("true # false");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is InvalidOperationException);
Assert.IsTrue(e.Message == "Invalid token started at: # false");
}
try
{
lexer.Analyze("true && ^");
Assert.Fail();
}
catch (Exception e)
{
Assert.IsTrue(e is InvalidOperationException);
Assert.IsTrue(e.Message == "Invalid token started at: ^");
}
}
static void Main(string[] args)
{
Lexer lex = new Lexer();
lex.AddSingleChar(LT.NONE, LT.COMMENT_1, '/', false, false);
lex.AddSingleChar(LT.COMMENT_1, LT.COMMENT_ML_1, '*', false, false);
lex.AddSingleChar(LT.COMMENT_ML_1, LT.COMMENT_ML_END_1, '*', false, false);
lex.FillRemaining(LT.COMMENT_ML_1, LT.COMMENT_ML_1, false, true);
lex.AddSingleChar(LT.COMMENT_ML_END_1, LT.COMMENT_ML, '/', true, false);
lex.AddSingleChar(LT.COMMENT_ML_END_1, LT.COMMENT_ML_END_1, '*', false, true);
lex.FillRemaining(LT.COMMENT_ML_END_1, LT.COMMENT_ML_1, false, true, "*");
lex.AddSingleChar(LT.COMMENT_1, LT.COMMENT_SL_1, '/', false, false);
lex.AddSingleChar(LT.COMMENT_SL_1, LT.COMMENT_SL, '\n', true, false);
lex.AddSingleChar(LT.COMMENT_SL_1, LT.COMMENT_SL, '\r', true, false);
lex.FillRemaining(LT.COMMENT_SL_1, LT.COMMENT_SL_1, false, true);
char[] letter = new[]
{
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '_'
};
char[] digit = new[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0' };
lex.AddSingleChar(LT.NONE, LT.STR_1, '"', false, false);
lex.AddSingleChar(LT.STR_1, LT.STRING_ESCAPE, '\\', false, false);
lex.AddSingleChar(LT.STRING_ESCAPE, LT.STR_1, 'n', false, true, "\n");
lex.AddSingleChar(LT.STRING_ESCAPE, LT.STR_1, 'r', false, true, "\r");
lex.AddSingleChar(LT.STRING_ESCAPE, LT.STR_1, 't', false, true, "\t");
lex.AddSingleChar(LT.STRING_ESCAPE, LT.STR_1, '\\', false, true);
lex.AddSingleChar(LT.STRING_ESCAPE, LT.STR_1, '"', false, true);
lex.AddByRange(LT.STR_1, LT.STR_1, ' ', '!', false, true);
lex.AddByRange(LT.STR_1, LT.STR_1, '#', '[', false, true);
lex.AddByRange(LT.STR_1, LT.STR_1, ']', '~', false, true);
lex.AddSingleChar(LT.STR_1, LT.LIT_STR, '"', true, false);
lex.FillRemaining(LT.STR_1, LT.NONE, false, true, "Invalid character in string");
lex.AddArray(LT.NONE, LT.IDENT, letter, true, true);
lex.AddArray(LT.IDENT, LT.IDENT, letter, true, true);
lex.AddArray(LT.IDENT, LT.IDENT, digit, true, true);
lex.AddSingleChar(LT.NONE, LT.CH_1, '\'', false, false);
lex.AddSingleChar(LT.CH_1, LT.CH_ESC, '\\', false, false);
lex.AddSingleChar(LT.CH_1, LT.NONE, '\'', false, false);
lex.AddByRange(LT.CH_1, LT.CH_2, ' ', '&', false, true);
lex.AddByRange(LT.CH_1, LT.CH_2, '(', '[', false, true);
lex.AddByRange(LT.CH_1, LT.CH_2, ']', '~', false, true);
lex.AddSingleChar(LT.CH_ESC, LT.CH_2, 'n', false, true, "\n");
lex.AddSingleChar(LT.CH_ESC, LT.CH_2, 'r', false, true, "\r");
lex.AddSingleChar(LT.CH_ESC, LT.CH_2, 't', false, true, "\t");
lex.AddSingleChar(LT.CH_ESC, LT.CH_2, '\\', false, true);
lex.AddSingleChar(LT.CH_ESC, LT.CH_2, '\'', false, true);
lex.AddSingleChar(LT.CH_2, LT.LIT_CHAR, '\'', true, false);
lex.AddArray(LT.NONE, LT.LIT_INT, digit, true, true);
lex.AddArray(LT.LIT_INT, LT.LIT_INT, digit, true, true);
lex.AddSingleChar(LT.LIT_INT, LT.LIT_FLOAT, '.', true, true);
lex.AddSingleChar(LT.NONE, LT.LIT_FLOAT_1, '.', false, true);
lex.AddArray(LT.LIT_FLOAT_1, LT.LIT_FLOAT, digit, true, true);
lex.AddArray(LT.LIT_FLOAT, LT.LIT_FLOAT, digit, true, true);
string numErr = "Non-numeric character found in numeric token";
lex.AddArray(LT.LIT_INT, LT.NONE, letter, false, true, numErr);
lex.AddArray(LT.LIT_FLOAT, LT.NONE, letter, false, true, numErr);
lex.AddString("struct", LT.KW_STRUCT);
lex.AddString("return", LT.KW_RETURN);
lex.AddString("break", LT.KW_BREAK);
lex.AddString("continue", LT.KW_CONT);
lex.AddString("if", LT.KW_IF);
lex.AddString("else", LT.KW_ELSE);
lex.AddString("true", LT.KW_TRUE);
lex.AddString("false", LT.KW_FALSE);
lex.AddString("while", LT.KW_WHILE);
lex.AddString("for", LT.KW_FOR);
lex.AddString("+", LT.OP_ADD);
lex.AddString("-", LT.OP_SUB);
lex.AddString("*", LT.OP_MUL);
lex.AddString("/", LT.OP_DIV);
lex.AddString("%", LT.OP_MOD);
lex.AddString("!", LT.OP_NEG);
lex.AddString(">", LT.OP_G);
lex.AddString("<", LT.OP_L);
lex.AddString(">=", LT.OP_GE);
lex.AddString("<=", LT.OP_LE);
lex.AddString("==", LT.OP_EQ);
lex.AddString("!=", LT.OP_NE);
lex.AddString("&", LT.OP_BAND);
lex.AddString("|", LT.OP_BOR);
lex.AddString("<<", LT.OP_BSHL);
lex.AddString(">>", LT.OP_BSHR);
lex.AddString("^", LT.OP_BXOR);
lex.AddString("~", LT.OP_BNOT);
lex.AddString("[", LT.OP_SQBRACKET_O);
lex.AddString("]", LT.OP_SQBRACKET_C);
lex.AddString("{", LT.OP_BRACE_O);
lex.AddString("}", LT.OP_BRACE_C);
lex.AddString("(", LT.OP_PAREN_O);
lex.AddString(")", LT.OP_PAREN_C);
lex.AddString(";", LT.OP_SC);
lex.AddString(",", LT.OP_COMMA);
lex.AddString("&&", LT.OP_AND);
lex.AddString("||", LT.OP_OR);
lex.AddString(".", LT.OP_PNT);
lex.AddString("=", LT.OP_ASSIGN);
lex.AddString(":", LT.OP_COLON);
//lex.FillRemaining(LT.NONE, LT.NONE, false, false);
var lexemes = lex.Analyze(string.Join(" ", args));
//if(lexemes != null)
//{
// Console.WriteLine(" ID | Line | Type | Value");
// Console.WriteLine("-------+-------+---------------------------+------------------");
// for (int i = 0; i < lexemes.Length; i++)
// {
// var l = lexemes[i];
// Console.WriteLine($" {i,5} | {l.Line,5} | {l.Type,-25} | {l.Value}");
// }
//}
File.WriteAllText("graph.dot", lex.GenerateDot());
var parser = new cmm.parser.Parser(lexemes);
parser.Parse();
//Console.WriteLine(
// JsonConvert.SerializeObject(
// parser.Parse(),
// Formatting.Indented,
// new JsonSerializerSettings
// {
// TypeNameHandling = TypeNameHandling.All,
// SerializationBinder = new SimpleNameBinder(),
// ReferenceLoopHandling = ReferenceLoopHandling.Serialize,
// PreserveReferencesHandling = PreserveReferencesHandling.Objects
// }
// )
//);
}