public JsonGrammar() { //Terminals var jstring = new StringLiteral("string", "\""); var jnumber = new NumberLiteral("number"); var comma = Symbol(","); //Nonterminals var jobject = new NonTerminal("Object"); var jarray = new NonTerminal("Array"); var jvalue = new NonTerminal("Value"); var jprop = new NonTerminal("Property"); var jproplist = new NonTerminal("PropertyList"); var jlist = new NonTerminal("List"); //Rules jvalue.Rule = jstring | jnumber | jobject | jarray | "true" | "false" | "null"; jobject.Rule = "{" + jproplist + "}"; jproplist.Rule = MakeStarRule(jproplist, comma, jprop); jprop.Rule = jstring + ":" + jvalue; jarray.Rule = "[" + jlist + "]"; jlist.Rule = MakeStarRule(jlist, comma, jvalue); //Set grammar root this.Root = jvalue; RegisterPunctuation("{", "}", "[", "]", ":", ","); this.MarkTransient(jvalue, jlist, jproplist); this.LanguageFlags = LanguageFlags.None; //.BubbleNodes; }
//http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf section 9.4.4 public static NumberLiteral CreateCSharpNumber(String name) { NumberLiteral term = new NumberLiteral(name); term.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.UInt32, TypeCode.Int64, TypeCode.UInt64 }; term.DefaultFloatType = TypeCode.Double; term.AddPrefix("0x", NumberFlags.Hex); term.AddSuffixCodes("u", TypeCode.UInt32, TypeCode.UInt64); term.AddSuffixCodes("l", TypeCode.Int64, TypeCode.UInt64); term.AddSuffixCodes("ul", TypeCode.UInt64); term.AddSuffixCodes("f", TypeCode.Single); term.AddSuffixCodes("d", TypeCode.Double); term.AddSuffixCodes("m", TypeCode.Decimal); return term; }
public ExpressionEvaluatorGrammar() { // 1. Terminals var number = new NumberLiteral("number"); var identifier = new IdentifierTerminal("identifier"); var comment = new CommentTerminal("comment", "#", "\n", "\r"); base.NonGrammarTerminals.Add(comment); // 2. Non-terminals var Variable = new NonTerminal("Variable", typeof(VarRefNode)); var Expr = new NonTerminal("Expr"); var Term = new NonTerminal("Term"); var BinExpr = new NonTerminal("BinExpr", typeof(BinExprNode)); var ParExpr = new NonTerminal("ParExpr"); var UnExpr = new NonTerminal("UnExpr", typeof(UnExprNode)); var UnOp = new NonTerminal("UnOp"); var BinOp = new NonTerminal("BinOp"); var AssignmentStmt = new NonTerminal("AssignmentStmt", typeof(AssigmentNode)); var Statement = new NonTerminal("Statement"); var ProgramLine = new NonTerminal("ProgramLine"); var Program = new NonTerminal("Program", typeof(StatementListNode)); // 3. BNF rules Variable.Rule = identifier; Expr.Rule = Term | UnExpr | BinExpr; Term.Rule = number | ParExpr | Variable; ParExpr.Rule = "(" + Expr + ")"; UnExpr.Rule = UnOp + Term; UnOp.Rule = Symbol("+") | "-"; BinExpr.Rule = Expr + BinOp + Expr; BinOp.Rule = Symbol("+") | "-" | "*" | "/" | "**"; AssignmentStmt.Rule = Variable + "=" + Expr; Statement.Rule = AssignmentStmt | Expr | Empty; ProgramLine.Rule = Statement + NewLine; Program.Rule = MakeStarRule(Program, ProgramLine); this.Root = Program; // Set grammar root // 4. Operators precedence RegisterOperators(1, "+", "-"); RegisterOperators(2, "*", "/"); RegisterOperators(3, Associativity.Right, "**"); RegisterPunctuation( "(", ")"); MarkTransient(Term, Expr, Statement); //automatically add NewLine before EOF so that our BNF rules work correctly when there's no final line break in source this.LanguageFlags |= LanguageFlags.NewLineBeforeEOF | LanguageFlags.SupportsInterpreter; }
public GWBasicGrammar() { #region Initialisation // BASIC is not case sensitive... this.CaseSensitive = false; // Define the Terminals var lineNumber = new NumberLiteral("NUMBER", NumberFlags.IntOnly); var fileNumber = new NumberLiteral("NUMBER", NumberFlags.IntOnly); var number = new NumberLiteral("NUMBER", NumberFlags.AllowStartEndDot); var variable = new IdentifierTerminal("Identifier", "$%!", string.Empty); var stringLiteral = new StringLiteral("STRING", "\"", StringFlags.None); //Important: do not add comment term to base.NonGrammarTerminals list - we do use this terminal in grammar rules var userFunctionName = variable; var comment = new CommentTerminal("Comment", "REM", "\n"); var short_comment = new CommentTerminal("ShortComment", "'", "\n"); var comma = Symbol(",", "comma"); var colon = Symbol(":", "colon"); var comma_opt = new NonTerminal("comma_opt"); comma_opt.Rule = Empty | ","; var semi_opt = new NonTerminal("semi_opt"); semi_opt.Rule = Empty | ";"; var pound_opt = new NonTerminal("pound_opt"); pound_opt.Rule = Empty | "#"; // Define the non-terminals var PROGRAM = new NonTerminal("PROGRAM"); var LINE = new NonTerminal("LINE"); var LINE_CONTENT = new NonTerminal("LINE_CONTENT"); var SHORT_COMMENT_OPT = new NonTerminal("SHORT_COMMENT_OPT"); var STATEMENT_LIST = new NonTerminal("STATEMENT_LIST"); var STATEMENT = new NonTerminal("STATEMENT"); var PRINT_STMT = new NonTerminal("PRINT_STMT"); var PRINT_LIST = new NonTerminal("PRINT_LIST"); var PRINT_ARG = new NonTerminal("PRINT_ARG"); var OPEN_STMT = new NonTerminal("OPEN_STMT"); var OPEN_STMT_MODE = new NonTerminal("OPEN_STMT_MODE"); var OPEN_STMT_ACCESS = new NonTerminal("OPEN_STMT_ACCESS"); var CLOSE_STMT = new NonTerminal("CLOSE_STMT"); var INPUT_STMT = new NonTerminal("INPUT_STMT"); var VARIABLES = new NonTerminal("VARIABLES"); var IF_STMT = new NonTerminal("IF_STMT"); var THEN_CLAUSE = new NonTerminal("THEN_CLAUSE"); var ELSE_CLAUSE_OPT = new NonTerminal("ELSE_CLAUSE_OPT"); //, typeof(AstNode)); var EXPR = new NonTerminal("EXPRESSION"); var EXPR_LIST = new NonTerminal("EXPRESSION_LIST"); var BINARY_OP = new NonTerminal("BINARY_OP"); var BINARY_EXPR = new NonTerminal("BINARY_EXPR"); var UNARY_EXPR = new NonTerminal("UNARY_EXPR"); var SIGN = new NonTerminal("SIGN"); var ASSIGN_STMT = new NonTerminal("ASSIGN_STMT"); var FOR_STMT = new NonTerminal("FOR_STMT"); var STEP_OPT = new NonTerminal("STEP_OPT"); var NEXT_STMT = new NonTerminal("NEXT_STMT"); var LOCATE_STMT = new NonTerminal("LOCATE_STMT"); var WHILE_STMT = new NonTerminal("WHILE_STMT"); var WEND_STMT = new NonTerminal("WEND_STMT"); var SWAP_STMT = new NonTerminal("SWAP_STMT"); var FUN_CALL = new NonTerminal("FUN_CALL"); var VARIABLE_OR_FUNCTION_EXPR = new NonTerminal("VARIABLE_OR_FUNCTION_EXPR"); var ARG_LIST = new NonTerminal("ARG_LIST"); var COMMENT_STMT = new NonTerminal("COMMENT_STMT"); var LINE_INPUT_STMT = new NonTerminal("LINE_INPUT_STMT"); var LINE_INPUT_POUND_STMT = new NonTerminal("LINE_INPUT_POUND_STMT"); var END_STMT = new NonTerminal("END_STMT"); var CLS_STMT = new NonTerminal("CLS_STMT", typeof(AstNode)); var CLEAR_STMT = new NonTerminal("CLEAR_STMT"); var DIM_STMT = new NonTerminal("DIM_STMT"); var DEF_FN_STMT = new NonTerminal("DEF_FN_STMT"); var GOTO_STMT = new NonTerminal("GOTO_STMT"); var GOSUB_STMT = new NonTerminal("GOSUB_STMT"); var RETURN_STMT = new NonTerminal("RETURN_STMT"); var ON_STMT = new NonTerminal("ON_STMT"); var LINE_NUMBERS = new NonTerminal("LINE_NUMBERS"); var RANDOMIZE_STMT = new NonTerminal("RANDOMIZE_STMT"); // set the PROGRAM to be the root node of BASIC programs. this.Root = PROGRAM; #endregion #region Grammar declaration // A program is a bunch of lines PROGRAM.Rule = MakePlusRule(PROGRAM, LINE); // A line can be an empty line, or it's a number followed by a statement list ended by a new-line. LINE.Rule = NewLine | lineNumber + LINE_CONTENT + SHORT_COMMENT_OPT + NewLine; // A statement list is 1 or more statements separated by the ':' character LINE_CONTENT.Rule = IF_STMT | COMMENT_STMT | STATEMENT_LIST; STATEMENT_LIST.Rule = MakePlusRule(STATEMENT_LIST, colon, STATEMENT); SHORT_COMMENT_OPT.Rule = short_comment | Empty; // A statement can be one of a number of types STATEMENT.Rule = ASSIGN_STMT | PRINT_STMT | INPUT_STMT | OPEN_STMT | CLOSE_STMT | LINE_INPUT_POUND_STMT | LINE_INPUT_STMT | LOCATE_STMT | CLS_STMT | END_STMT | CLEAR_STMT | DIM_STMT | DEF_FN_STMT | SWAP_STMT | RANDOMIZE_STMT | GOSUB_STMT | RETURN_STMT | GOTO_STMT | ON_STMT | FOR_STMT | NEXT_STMT | WHILE_STMT | WEND_STMT; // The different statements are defined here PRINT_STMT.Rule = "print" + PRINT_LIST; PRINT_LIST.Rule = MakeStarRule(PRINT_LIST, null, PRINT_ARG); PRINT_ARG.Rule = EXPR + semi_opt; INPUT_STMT.Rule = "input" + semi_opt + stringLiteral + ";" + VARIABLES; OPEN_STMT.Rule = "open" + EXPR + (Empty | "for" + OPEN_STMT_MODE) + (Empty | "access" + OPEN_STMT_ACCESS) + "as" + pound_opt + fileNumber; OPEN_STMT_ACCESS.Rule = "read" + (Empty | "write") | "write"; OPEN_STMT_MODE.Rule = Symbol("o") | "i" | "a" | "output" | "input" | "append"; CLOSE_STMT.Rule = "close" + pound_opt + number; LINE_INPUT_STMT.Rule = Symbol("line") + "input" + semi_opt + stringLiteral + ";" + VARIABLE_OR_FUNCTION_EXPR; LINE_INPUT_POUND_STMT.Rule = Symbol("line") + "input" + Symbol("#") + fileNumber + comma + VARIABLE_OR_FUNCTION_EXPR; DIM_STMT.Rule = "dim" + VARIABLES; DEF_FN_STMT.Rule = "def" + userFunctionName + (Empty | "(" + ARG_LIST + ")") + "=" + EXPR; VARIABLES.Rule = VARIABLE_OR_FUNCTION_EXPR | VARIABLE_OR_FUNCTION_EXPR + "," + VARIABLES; IF_STMT.Rule = "if" + EXPR + THEN_CLAUSE + ELSE_CLAUSE_OPT; THEN_CLAUSE.Rule = "then" + STATEMENT_LIST | "goto" + lineNumber; //Inject PreferShift hint here to explicitly set shift as preferred action. Suppresses warning message about conflict. ELSE_CLAUSE_OPT.Rule = Empty | PreferShiftHere() + "else" + STATEMENT_LIST; GOTO_STMT.Rule = "goto" + lineNumber; GOSUB_STMT.Rule = "gosub" + lineNumber; RETURN_STMT.Rule = "return"; ON_STMT.Rule = "on" + EXPR + (Symbol("goto") | "gosub") + LINE_NUMBERS; LINE_NUMBERS.Rule = lineNumber + (Empty | "," + LINE_NUMBERS); ASSIGN_STMT.Rule = VARIABLE_OR_FUNCTION_EXPR + "=" + EXPR; LOCATE_STMT.Rule = "locate" + EXPR + comma + EXPR; SWAP_STMT.Rule = "swap" + EXPR + comma + EXPR; END_STMT.Rule = "end"; CLS_STMT.Rule = "cls"; CLEAR_STMT.Rule = Symbol("clear") + comma + (Empty | number) + (Empty | comma + number) | "clear" + number | "clear"; COMMENT_STMT.Rule = comment | short_comment; RANDOMIZE_STMT.Rule = "randomize" + EXPR; // An expression is a number, or a variable, a string, or the result of a binary comparison. EXPR.Rule = number | variable | FUN_CALL | stringLiteral | BINARY_EXPR | "(" + EXPR + ")" | UNARY_EXPR; BINARY_EXPR.Rule = EXPR + BINARY_OP + EXPR; UNARY_EXPR.Rule = SIGN + EXPR; SIGN.Rule = Symbol("-") | "+"; //Inject PreferShift hint here to explicitly set shift as preferred action. Suppresses warning message about conflict. FUN_CALL.Rule = variable + PreferShiftHere() + "(" + ARG_LIST + ")"; VARIABLE_OR_FUNCTION_EXPR.Rule = variable | FUN_CALL; BINARY_OP.Rule = Symbol("+") | "^" | "-" | "*" | "/" | "=" | "<=" | ">=" | "<" | ">" | "<>" | "and" | "or"; //let's do operator precedence right here RegisterOperators(60, "^"); RegisterOperators(50, "*", "/"); RegisterOperators(40, "+", "-"); RegisterOperators(30, "=", "<=", ">=", "<", ">", "<>"); RegisterOperators(20, "and", "or"); EXPR_LIST.Rule = MakeStarRule(EXPR_LIST, null, EXPR); FOR_STMT.Rule = "for" + ASSIGN_STMT + "to" + EXPR + STEP_OPT; STEP_OPT.Rule = Empty | "step" + EXPR; NEXT_STMT.Rule = "next" + VARIABLES | "next"; WHILE_STMT.Rule = "while" + EXPR; WEND_STMT.Rule = "wend"; //TODO: check number of arguments for particular function in node constructor ARG_LIST.Rule = MakePlusRule(ARG_LIST, comma, EXPR); #endregion #region Punctuation RegisterPunctuation("(", ")", ",", ";"); #endregion }
//http://docs.python.org/ref/numbers.html public static NumberLiteral CreatePythonNumber(String name) { NumberLiteral term = new NumberLiteral(name, NumberFlags.AllowStartEndDot); //default int types are Integer (32bit) -> LongInteger (BigInt); Try Int64 before BigInt: Better performance? term.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt }; // term.DefaultFloatType = TypeCode.Double; -- it is default //float type is implementation specific, thus try decimal first (higher precision) //term.DefaultFloatTypes = new TypeCode[] { TypeCode.Decimal, TypeCode.Double }; term.AddPrefix("0x", NumberFlags.Hex); term.AddPrefix("0", NumberFlags.Octal); term.AddSuffixCodes("L", TypeCode.Int64, NumberLiteral.TypeCodeBigInt); term.AddSuffixCodes("J", NumberLiteral.TypeCodeImaginary); return term; }
//http://www.microsoft.com/downloads/details.aspx?FamilyId=6D50D709-EAA4-44D7-8AF3-E14280403E6E&displaylang=en section 2 public static NumberLiteral CreateVbNumber(String name) { NumberLiteral term = new NumberLiteral(name); term.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64 }; //term.DefaultFloatType = TypeCode.Double; it is default term.AddPrefix("&H", NumberFlags.Hex); term.AddPrefix("&O", NumberFlags.Octal); term.AddSuffixCodes("S", TypeCode.Int16); term.AddSuffixCodes("I", TypeCode.Int32); term.AddSuffixCodes("%", TypeCode.Int32); term.AddSuffixCodes("L", TypeCode.Int64); term.AddSuffixCodes("&", TypeCode.Int64); term.AddSuffixCodes("D", TypeCode.Decimal); term.AddSuffixCodes("@", TypeCode.Decimal); term.AddSuffixCodes("F", TypeCode.Single); term.AddSuffixCodes("!", TypeCode.Single); term.AddSuffixCodes("R", TypeCode.Double); term.AddSuffixCodes("#", TypeCode.Double); term.AddSuffixCodes("US", TypeCode.UInt16); term.AddSuffixCodes("UI", TypeCode.UInt32); term.AddSuffixCodes("UL", TypeCode.UInt64); return term; }
//Note - this is incomplete implementation; need to add functionality to NumberTerminal class to support type detection based // on exponent symbol. // From R6RS: // ... representations of number objects may be written with an exponent marker that indicates the desired precision // of the inexact representation. The letters s, f, d, and l specify the use of short, single, double, and long precision, respectively. public static NumberLiteral CreateSchemeNumber(String name) { NumberLiteral term = new NumberLiteral(name); term.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt }; term.DefaultFloatType = TypeCode.Double; // it is default term.ExponentSymbols = "sfdl"; term.AddPrefix("#b", NumberFlags.Binary); term.AddPrefix("#o", NumberFlags.Octal); term.AddPrefix("#x", NumberFlags.Hex); term.AddPrefix("#d", NumberFlags.None); term.AddPrefix("#i", NumberFlags.None); // inexact prefix, has no effect term.AddPrefix("#e", NumberFlags.None); // exact prefix, has no effect term.AddSuffixCodes("J", NumberLiteral.TypeCodeImaginary); return term; }
public SqlGrammar() { //Terminals var comment = new CommentTerminal("comment", "/*", "*/"); var lineComment = new CommentTerminal("line_comment", "--", "\n", "\r\n"); NonGrammarTerminals.Add(comment); NonGrammarTerminals.Add(lineComment); var number = new NumberLiteral("number"); var string_literal = new StringLiteral("string", "'", StringFlags.AllowsDoubledQuote); var name = new IdentifierTerminal("name"); var name_ext = TerminalFactory.CreateSqlExtIdentifier("name_ext"); var comma = Symbol(","); var dot = Symbol("."); var CREATE = Symbol("CREATE"); var NULL = Symbol("NULL"); var NOT = Symbol("NOT"); var UNIQUE = Symbol("UNIQUE"); var WITH = Symbol("WITH"); var TABLE = Symbol("TABLE"); var ALTER = Symbol("ALTER"); var ADD = Symbol("ADD"); var COLUMN = Symbol("COLUMN"); var DROP = Symbol("DROP"); var CONSTRAINT = Symbol("CONSTRAINT"); var INDEX = Symbol("INDEX"); var ON = Symbol("ON"); var KEY = Symbol("KEY"); var PRIMARY = Symbol("PRIMARY"); var INSERT = Symbol("INSERT"); var INTO = Symbol("INTO"); var UPDATE = Symbol("UPDATE"); var SET = Symbol("SET"); var VALUES = Symbol("VALUES"); var DELETE = Symbol("DELETE"); var SELECT = Symbol("SELECT"); var FROM = Symbol("FROM"); var AS = Symbol("AS"); var COUNT = Symbol("COUNT"); var JOIN = Symbol("JOIN"); var BY = Symbol("BY"); //Non-terminals var Id = new NonTerminal("Id"); var Id_simple = new NonTerminal("id_simple"); var stmt = new NonTerminal("stmt"); var createTableStmt = new NonTerminal("createTableStmt"); var createIndexStmt = new NonTerminal("createIndexStmt"); var alterStmt = new NonTerminal("alterStmt"); var dropTableStmt = new NonTerminal("dropTableStmt"); var dropIndexStmt = new NonTerminal("dropIndexStmt"); var selectStmt = new NonTerminal("selectStmt"); var insertStmt = new NonTerminal("insertStmt"); var updateStmt = new NonTerminal("updateStmt"); var deleteStmt = new NonTerminal("deleteStmt"); var fieldDef = new NonTerminal("fieldDef"); var fieldDefList = new NonTerminal("fieldDefList"); var nullSpecOpt = new NonTerminal("nullSpecOpt"); var typeName = new NonTerminal("typeName"); var typeSpec = new NonTerminal("typeSpec"); var typeParamsOpt = new NonTerminal("typeParams"); var constraintDef = new NonTerminal("constraintDef"); var constraintListOpt = new NonTerminal("constraintListOpt"); var constraintTypeOpt = new NonTerminal("constraintTypeOpt"); var idlist = new NonTerminal("idlist"); var idlistPar = new NonTerminal("idlistPar"); var uniqueOpt = new NonTerminal("uniqueOpt"); var orderList = new NonTerminal("orderList"); var orderMember = new NonTerminal("orderMember"); var orderDirOpt = new NonTerminal("orderDirOpt"); var withClauseOpt = new NonTerminal("withClauseOpt"); var alterCmd = new NonTerminal("alterCmd"); var insertData = new NonTerminal("insertData"); var intoOpt = new NonTerminal("intoOpt"); var assignList = new NonTerminal("assignList"); var whereClauseOpt = new NonTerminal("whereClauseOpt"); var assignment = new NonTerminal("assignment"); var expression = new NonTerminal("expression"); var exprList = new NonTerminal("exprList"); var selRestrOpt = new NonTerminal("selRestrOpt"); var selList = new NonTerminal("selList"); var intoClauseOpt = new NonTerminal("intoClauseOpt"); var fromClauseOpt = new NonTerminal("fromClauseOpt"); var groupClauseOpt = new NonTerminal("groupClauseOpt"); var havingClauseOpt = new NonTerminal("havingClauseOpt"); var orderClauseOpt = new NonTerminal("orderClauseOpt"); var columnItemList = new NonTerminal("columnItemList"); var columnItem = new NonTerminal("columnItem"); var columnSource = new NonTerminal("columnSource"); var asOpt = new NonTerminal("asOpt"); var aliasOpt = new NonTerminal("aliasOpt"); var aggregate = new NonTerminal("aggregate"); var aggregateArg = new NonTerminal("aggregateArg"); var aggregateName = new NonTerminal("aggregateName"); var tuple = new NonTerminal("tuple"); var joinChainOpt = new NonTerminal("joinChainOpt"); var joinKindOpt = new NonTerminal("joinKindOpt"); var term = new NonTerminal("term"); var unExpr = new NonTerminal("unExpr"); var unOp = new NonTerminal("unOp"); var binExpr = new NonTerminal("binExpr"); var binOp = new NonTerminal("binOp"); var betweenExpr = new NonTerminal("betweenExpr"); var inExpr = new NonTerminal("inExpr"); var parSelectStmt = new NonTerminal("parSelectStmt"); var notOpt = new NonTerminal("notOpt"); var funCall = new NonTerminal("funCall"); var stmtLine = new NonTerminal("stmtLine"); var semiOpt = new NonTerminal("semiOpt"); var stmtList = new NonTerminal("stmtList"); var funArgs = new NonTerminal("funArgs"); var inStmt = new NonTerminal("inStmt"); //BNF Rules this.Root = stmtList; stmtLine.Rule = stmt + semiOpt; semiOpt.Rule = Empty | ";"; stmtList.Rule = MakePlusRule(stmtList, stmtLine); //ID Id_simple.Rule = name | name_ext; Id.Rule = MakePlusRule(Id, dot, Id_simple); stmt.Rule = createTableStmt | createIndexStmt | alterStmt | dropTableStmt | dropIndexStmt | selectStmt | insertStmt | updateStmt | deleteStmt | "GO" ; //Create table createTableStmt.Rule = CREATE + TABLE + Id + "(" + fieldDefList + ")" + constraintListOpt; fieldDefList.Rule = MakePlusRule(fieldDefList, comma, fieldDef); fieldDef.Rule = Id + typeName + typeParamsOpt + nullSpecOpt; nullSpecOpt.Rule = NULL | NOT + NULL | Empty; typeName.Rule = Symbol("BIT") | "DATE" | "TIME" | "TIMESTAMP" | "DECIMAL" | "REAL" | "FLOAT" | "SMALLINT" | "INTEGER" | "INTERVAL" | "CHARACTER" // MS SQL types: | "DATETIME" | "INT" | "DOUBLE" | "CHAR" | "NCHAR" | "VARCHAR" | "NVARCHAR" | "IMAGE" | "TEXT" | "NTEXT"; typeParamsOpt.Rule = "(" + number + ")" | "(" + number + comma + number + ")" | Empty; constraintDef.Rule = CONSTRAINT + Id + constraintTypeOpt; constraintListOpt.Rule = MakeStarRule(constraintListOpt, constraintDef ); constraintTypeOpt.Rule = PRIMARY + KEY + idlistPar | UNIQUE + idlistPar | NOT + NULL + idlistPar | "Foreign" + KEY + idlistPar + "References" + Id + idlistPar; idlistPar.Rule = "(" + idlist + ")"; idlist.Rule = MakePlusRule(idlist, comma, Id); //Create Index createIndexStmt.Rule = CREATE + uniqueOpt + INDEX + Id + ON + Id + orderList + withClauseOpt; uniqueOpt.Rule = Empty | UNIQUE; orderList.Rule = MakePlusRule(orderList, comma, orderMember); orderMember.Rule = Id + orderDirOpt; orderDirOpt.Rule = Empty | "ASC" | "DESC"; withClauseOpt.Rule = Empty | WITH + PRIMARY | WITH + "Disallow" + NULL | WITH + "Ignore" + NULL; //Alter alterStmt.Rule = ALTER + TABLE + Id + alterCmd; alterCmd.Rule = ADD + COLUMN + fieldDefList + constraintListOpt | ADD + constraintDef | DROP + COLUMN + Id | DROP + CONSTRAINT + Id; //Drop stmts dropTableStmt.Rule = DROP + TABLE + Id; dropIndexStmt.Rule = DROP + INDEX + Id + ON + Id; //Insert stmt insertStmt.Rule = INSERT + intoOpt + Id + idlistPar + insertData; insertData.Rule = selectStmt | VALUES + "(" + exprList + ")"; intoOpt.Rule = Empty | INTO; //Into is optional in MSSQL //Update stmt updateStmt.Rule = UPDATE + Id + SET + assignList + whereClauseOpt; assignList.Rule = MakePlusRule(assignList, comma, assignment); assignment.Rule = Id + "=" + expression; //Delete stmt deleteStmt.Rule = DELETE + FROM + Id + whereClauseOpt; //Select stmt selectStmt.Rule = SELECT + selRestrOpt + selList + intoClauseOpt + fromClauseOpt + whereClauseOpt + groupClauseOpt + havingClauseOpt + orderClauseOpt; selRestrOpt.Rule = Empty | "ALL" | "DISTINCT"; selList.Rule = columnItemList | "*"; columnItemList.Rule = MakePlusRule(columnItemList, comma, columnItem); columnItem.Rule = columnSource + aliasOpt; aliasOpt.Rule = Empty | asOpt + Id; asOpt.Rule = Empty | AS; columnSource.Rule = aggregate | Id; aggregate.Rule = aggregateName + "(" + aggregateArg + ")"; aggregateArg.Rule = expression | "*"; aggregateName.Rule = COUNT | "Avg" | "Min" | "Max" | "StDev" | "StDevP" | "Sum" | "Var" | "VarP"; intoClauseOpt.Rule = Empty | INTO + Id; fromClauseOpt.Rule = Empty | FROM + idlist + joinChainOpt; joinChainOpt.Rule = Empty | joinKindOpt + JOIN + idlist + ON + Id + "=" + Id; joinKindOpt.Rule = Empty | "INNER" | "LEFT" | "RIGHT"; whereClauseOpt.Rule = Empty | "WHERE" + expression; groupClauseOpt.Rule = Empty | "GROUP" + BY + idlist; havingClauseOpt.Rule = Empty | "HAVING" + expression; orderClauseOpt.Rule = Empty | "ORDER" + BY + orderList; //Expression exprList.Rule = MakePlusRule(exprList, comma, expression); expression.Rule = term | unExpr | binExpr;// | betweenExpr; //-- BETWEEN doesn't work - yet; brings a few parsing conflicts term.Rule = Id | string_literal | number | funCall | tuple | parSelectStmt;// | inStmt; tuple.Rule = "(" + exprList + ")"; parSelectStmt.Rule = "(" + selectStmt + ")"; unExpr.Rule = unOp + term; unOp.Rule = NOT | "+" | "-" | "~"; binExpr.Rule = expression + binOp + expression; binOp.Rule = Symbol("+") | "-" | "*" | "/" | "%" //arithmetic | "&" | "|" | "^" //bit | "=" | ">" | "<" | ">=" | "<=" | "<>" | "!=" | "!<" | "!>" | "AND" | "OR" | "LIKE" | NOT + "LIKE" | "IN" | NOT + "IN" ; betweenExpr.Rule = expression + notOpt + "BETWEEN" + expression + "AND" + expression; notOpt.Rule = Empty | NOT; //funCall covers some psedo-operators and special forms like ANY(...), SOME(...), ALL(...), EXISTS(...), IN(...) funCall.Rule = name + "(" + funArgs + ")"; funArgs.Rule = selectStmt | exprList; inStmt.Rule = expression + "IN" + "(" + exprList + ")"; //Operators this.CaseSensitive = false; RegisterOperators(10, "*", "/", "%"); RegisterOperators(9, "+", "-"); RegisterOperators(8, "=" , ">" , "<" , ">=" , "<=" , "<>" , "!=" , "!<" , "!>"); RegisterOperators(7, "^", "&", "|"); RegisterOperators(6, "NOT"); RegisterOperators(5, "AND"); RegisterOperators(4, "OR", "LIKE", "IN"); RegisterPunctuation(",", "(", ")"); RegisterPunctuation(semiOpt); base.MarkTransient(stmt, Id_simple, asOpt, aliasOpt, stmtLine); AddKeywords("SELECT", "CREATE", "ALTER", "UPDATE", "INSERT", "DELETE", "FROM", "WHERE", "GROUP", "ORDER", "BY", "INNER", "LEFT", "RIGHT", "JOIN", "ON"); }
// It is loosely based on R6RS specs. // See Grammar Errors tab in GrammarExplorer for remaining conflicts. public SchemeGrammar() { #region Terminals ConstantTerminal Constant = new ConstantTerminal("Constant"); Constant.Add("#T", 1); Constant.Add("#t", 1); Constant.Add("#F", null); Constant.Add("#f", null); Constant.Add("'()", null); Constant.Add(@"#\nul", '\u0000'); Constant.Add(@"#\alarm", '\u0007'); Constant.Add(@"#\backspace", '\b'); Constant.Add(@"#\tab", '\t'); Constant.Add(@"#\linefeed", '\n'); Constant.Add(@"#\vtab", '\v'); Constant.Add(@"#\page", '\f'); Constant.Add(@"#\return", '\r'); Constant.Add(@"#\esc", '\u001B'); Constant.Add(@"#\space", ' '); Constant.Add(@"#\delete", '\u007F'); // TODO: build SchemeCharLiteral // the following is nonsense, just to put something there var charLiteral = new StringLiteral("Char", "'", StringFlags.None); var stringLiteral = new StringLiteral("String", "\"", StringFlags.AllowsAllEscapes); //Identifiers. Note: added "-", just to allow IDs starting with "->" var SimpleIdentifier = new IdentifierTerminal("SimpleIdentifier", "_+-*/.@?!<>=", "_+-*/.@?!<>=$%&:^~"); // name extraChars extraFirstChars var Number = TerminalFactory.CreateSchemeNumber("Number"); var Byte = new NumberLiteral("Byte", NumberFlags.IntOnly); //Comments Terminal Comment = new CommentTerminal("Comment", "#|", "|#"); Terminal LineComment = new CommentTerminal("LineComment", ";", "\n"); NonGrammarTerminals.Add(Comment); //add comments explicitly to this list as it is not reachable from Root NonGrammarTerminals.Add(LineComment); #endregion #region NonTerminals var Module = new NonTerminal("Module"); var Library = new NonTerminal("Library"); var LibraryList = new NonTerminal("Library+"); var Script = new NonTerminal("Script"); var Abbreviation = new NonTerminal("Abbreviation"); var Vector = new NonTerminal("Vector"); var ByteList = new NonTerminal("ByteList"); var ByteVector = new NonTerminal("ByteVector"); var Datum = new NonTerminal("Datum"); //Datum in R6RS terms var DatumOpt = new NonTerminal("DatumOpt"); //Datum in R6RS terms var DatumList = new NonTerminal("Datum+", typeof(StatementListNode)); var DatumListOpt = new NonTerminal("Datum*", typeof(StatementListNode)); var Statement = new NonTerminal("Statement"); var Atom = new NonTerminal("Atom"); var CompoundDatum = new NonTerminal("CompoundDatum"); var AbbrevPrefix = new NonTerminal("AbbrevPrefix"); var LibraryName = new NonTerminal("LibraryName"); var LibraryBody = new NonTerminal("LibraryBody"); var ImportSection = new NonTerminal("ImportSection"); var ExportSection = new NonTerminal("ExportSection"); var ImportSpec = new NonTerminal("ImportSpec"); var ImportSpecList = new NonTerminal("ImportSpecList"); var ExportSpec = new NonTerminal("ExportSpec"); var ExportSpecList = new NonTerminal("ExportSpecList"); var LP = new NonTerminal("LP"); //"(" or "[" var RP = new NonTerminal("RP"); // ")" or "]" var Identifier = new NonTerminal("Identifier", typeof(VarRefNode)); var IdentifierList = new NonTerminal("IdentifierList"); var IdentifierListOpt = new NonTerminal("IdentifierListOpt"); var PeculiarIdentifier = new NonTerminal("PeculiarIdentifier"); var LibraryVersion = new NonTerminal("LibraryVersion"); var VersionListOpt = new NonTerminal("VersionListOpt"); var FunctionCall = new NonTerminal("FunctionCall", CreateFunctionCallNode); var FunctionRef = new NonTerminal("FunctionRef"); var SpecialForm = new NonTerminal("SpecialForm"); var DefineVarForm = new NonTerminal("DefineVarForm", CreateDefineVarNode); var DefineFunForm = new NonTerminal("DefineFunForm", CreateDefineFunNode); var LambdaForm = new NonTerminal("LambdaForm", CreateLambdaNode); var IfForm = new NonTerminal("IfForm", CreateIfThenElseNode); var CondForm = new NonTerminal("CondForm", CreateCondFormNode); var CondClause = new NonTerminal("CondClause", CreateCondClauseNode); var CondClauseList = new NonTerminal("CondClauseList"); var CondElseOpt = new NonTerminal("CondElseOpt"); var BeginForm = new NonTerminal("BeginForm", CreateBeginNode); var LetForm = new NonTerminal("LetForm"); //not implemented var LetRecForm = new NonTerminal("LetRecForm"); //not implemented var LetPair = new NonTerminal("LetPair"); var LetPairList = new NonTerminal("LetPairList"); #endregion #region Rules // // Using optional elements in Scheme grammar brings some nasty conflicts - by default the parser selects "shift" over reduce // which leads to failure to parse simple programs without libraries and export/import sections. // This trouble comes from that the fact that Scheme has soooooo many parenthesis. Therefore, using a single next symbol // as a lookahead (as it happens in LALR parsing) doesn't help much - the next symbol is almost always a parenthesis, // not some meaningful symbol. That's why in the following expressions I had to use explicit listing of variants, // instead of simply marking some elements as optional (see Module, Script, LibraryBody elements) - this clears the conflicts // but would make node construction more difficult. base.Root = Module; LP.Rule = Symbol("(") | "["; //R6RS allows mix & match () and [] RP.Rule = Symbol(")") | "]"; // Module.Rule = LibraryListOpt + Script; -- this brings conflicts Module.Rule = LibraryList + Script | Script; LibraryList.Rule = MakePlusRule(LibraryList, Library); Script.Rule = ImportSection + DatumList | DatumList; //Library // the following doesn't work - brings conflicts that incorrectly resolved by default shifting //Library.Rule = LP + "library" + LibraryName + ExportSectionOpt + ImportSectionOpt + DatumListOpt + RP; Library.Rule = LP + "library" + LibraryName + LibraryBody + RP; //Note - we should be using DatumListOpt, but that brings 2 conflicts, so for now it is just DatumList //Note that the following style of BNF expressions is strongly discouraged - all productions should be of the same length, // so that the process of mapping child nodes to parent's properties is straightforward. LibraryBody.Rule = ExportSection + ImportSection + DatumList | ExportSection + DatumList | ImportSection + DatumList | DatumList; LibraryName.Rule = LP + IdentifierList + LibraryVersion.Q() + RP; LibraryVersion.Rule = LP + VersionListOpt + RP; //zero or more subversion numbers VersionListOpt.Rule = MakeStarRule(VersionListOpt, Number); ExportSection.Rule = LP + "export" + ExportSpecList + RP; ImportSection.Rule = LP + "import" + ImportSpecList + RP; ExportSpecList.Rule = MakePlusRule(ExportSpecList, ExportSpec); ImportSpecList.Rule = MakePlusRule(ImportSpecList, ImportSpec); ExportSpec.Rule = Identifier | LP + "rename" + LP + Identifier + Identifier + RP + RP; ImportSpec.Rule = LP + Identifier + RP; // - much more complex in R6RS //Datum Datum.Rule = Atom | CompoundDatum; DatumOpt.Rule = Empty | Datum; DatumList.Rule = MakePlusRule(DatumList, Datum); DatumListOpt.Rule = MakeStarRule(DatumListOpt, Datum); Atom.Rule = Number | Identifier | stringLiteral | Constant | charLiteral | "."; CompoundDatum.Rule = Statement | Abbreviation | Vector | ByteVector; Identifier.Rule = SimpleIdentifier | PeculiarIdentifier; IdentifierList.Rule = MakePlusRule(IdentifierList, Identifier); IdentifierListOpt.Rule = MakeStarRule(IdentifierListOpt, Identifier); //TODO: create PeculiarIdentifier custom terminal instead of var // or just custom SchemeIdentifier terminal PeculiarIdentifier.Rule = Symbol("+") | "-" | "..."; // |"->" + subsequent; (should be!) Abbreviation.Rule = AbbrevPrefix + Datum; AbbrevPrefix.Rule = Symbol("'") | "`" | ",@" | "," | "#'" | "#`" | "#,@" | "#,"; Vector.Rule = "#(" + DatumListOpt + ")"; ByteVector.Rule = "#vu8(" + ByteList + ")"; ByteList.Rule = MakeStarRule(ByteList, Byte); Statement.Rule = FunctionCall | SpecialForm; FunctionCall.Rule = LP + FunctionRef + DatumListOpt + RP; FunctionRef.Rule = Identifier | Statement; SpecialForm.Rule = DefineVarForm | DefineFunForm | LambdaForm | IfForm | CondForm | BeginForm | LetForm | LetRecForm; DefineVarForm.Rule = LP + "define" + Identifier + Datum + RP; DefineFunForm.Rule = LP + "define" + LP + Identifier + IdentifierListOpt + RP + DatumList + RP; LambdaForm.Rule = LP + "lambda" + LP + IdentifierListOpt + RP + DatumList + RP; IfForm.Rule = LP + "if" + Datum + Datum + DatumOpt + RP; CondForm.Rule = LP + "cond" + CondClauseList + CondElseOpt + RP; CondClauseList.Rule = MakePlusRule(CondClauseList, CondClause); CondClause.Rule = LP + Datum + DatumList + RP; CondElseOpt.Rule = Empty | LP + "else" + DatumList + RP; LetForm.Rule = LP + "let" + LP + LetPairList + RP + DatumList + RP; LetRecForm.Rule = LP + "letrec" + LP + LetPairList + RP + DatumList + RP; BeginForm.Rule = LP + "begin" + DatumList + RP; LetPairList.Rule = MakePlusRule(LetPairList, LetPair); LetPair.Rule = LP + Identifier + Datum + RP; #endregion //Register brace pairs RegisterBracePair("(", ")"); RegisterBracePair("[", "]"); RegisterPunctuation(LP, RP); //Filters and other stuff BraceMatchFilter filter = new BraceMatchFilter(); TokenFilters.Add(filter); //Scheme is tail-recursive language base.LanguageFlags = LanguageFlags.BubbleNodes | LanguageFlags.TailRecursive | LanguageFlags.SupportsInterpreter | LanguageFlags.SupportsConsole; //keywords - just for colorizer base.AddKeywords("define", "lambda", "cond", "if", "begin", "let"); }