public static Declaration Parse(String Header) { var tokenIterator = new TokenStream(new StringIterator(Header), new ParseContext()); var headerTerms = EtcScriptLib.Parse.ParseMacroDeclarationHeader(tokenIterator, EtcScriptLib.Parse.DeclarationHeaderTerminatorType.StreamEnd); var r = new Declaration(); r.ReturnTypeName = "VOID"; if (!tokenIterator.AtEnd() && tokenIterator.Next().Type == TokenType.Colon) { tokenIterator.Advance(); if (tokenIterator.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", tokenIterator); r.ReturnTypeName = tokenIterator.Next().Value.ToUpper(); tokenIterator.Advance(); } if (!tokenIterator.AtEnd()) throw new CompileError("Header did not end when expected"); r.Terms = headerTerms; r.Body = new LambdaBlock(null); return r; }
private static Ast.Return ParseReturnStatement( TokenStream Stream, ParseContext Context) { var start = Stream.Next(); if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "RETURN") throw new CompileError("[007] Impossible error: Return parse entered, no return found.", Stream); Stream.Advance(); var r = new Ast.Return(start); if (Stream.Next().Type != TokenType.Semicolon) r.Value = ParseExpression(Stream, Context, TokenType.Semicolon); if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[008] Expected ;", Stream); Stream.Advance(); return r; }
public void Style(TokenStream Stream) { this.Stream = Stream; this.FoldLevel = 1; while (!Stream.AtEnd()) { if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION" || Stream.Next().Value.ToUpper() == "TEST") ParseMacroDeclaration(); else if (Stream.Next().Value.ToUpper() == "RULE") ParseRuleDeclaration(); else if (Stream.Next().Value.ToUpper() == "TYPE") ParseTypeDeclaration(); else if (Stream.Next().Value.ToUpper() == "GLOBAL") ParseGlobalDeclaration(); else if (Stream.Next().Value.ToUpper() == "INCLUDE") { Style(TokenStyle.Keyword); Expect(TokenType.String); } else if (Stream.Next().Value.ToUpper() == "DEFAULT") { Style(TokenStyle.Keyword); Expect(TokenType.Identifier, t => t.Value.ToUpper() == "OF"); ParseRuleDeclaration(); } else Stream.Advance(); } }
private static Ast.Let ParseLetStatement( TokenStream Stream, ParseContext Context) { var start = Stream.Next(); if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "LET") throw new CompileError("[000] Impossible error: Let parse entered, no let found.", Stream); Stream.Advance(); var LHS = ParseTerm(Stream, Context); if (Stream.Next().Type != TokenType.Operator || Stream.Next().Value != "=") throw new CompileError("[001] Expected '='", Stream); Stream.Advance(); var RHS = ParseExpression(Stream, Context, TokenType.Semicolon); if (!IsEndOfStatement(Stream)) throw new CompileError("[002] Expected ;", Stream); Stream.Advance(); return new Ast.Let(start, LHS, RHS); }
private static Ast.Node ParseOptionalDot( TokenStream Stream, Ast.Node LHS, ParseContext Context) { if (Stream.AtEnd()) return LHS; if (Stream.Next().Type == TokenType.Dot) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[00F] Dot operator must be followed by identifier", Stream); var RHS = Stream.Next().Value; var MA = new Ast.MemberAccess(LHS.Source); MA.Object = LHS; MA.Name = RHS.ToUpper(); Stream.Advance(); return ParseOptionalDot(Stream, MA, Context); } else if (Stream.Next().Type == TokenType.At) { Stream.Advance(); var RHS = ParseTerm(Stream, Context); var I = new Ast.Indexer(LHS.Source); I.Object = LHS; I.Index = RHS; return ParseOptionalDot(Stream, I, Context); } else if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[013] Expected identifier", Stream); var C = new Ast.Cast(Stream.Next(), LHS, Stream.Next().Value.ToUpper()); Stream.Advance(); return ParseOptionalDot(Stream, C, Context); } else return LHS; }
//Implements http://en.wikipedia.org/wiki/Operator-precedence_parser private static Ast.Node ParseExpression( Ast.Node lhs, TokenStream state, ParseContext operators, int minimum_precedence, Predicate<TokenStream> IsTerminal) { while (true) { if (IsTerminal(state)) return lhs; if (state.Next().Type != TokenType.Operator) throw new CompileError("[01C] Expected operator", state.Next()); var precedence = operators.FindPrecedence(state.Next().Value); if (precedence < minimum_precedence) return lhs; var op = state.Next(); state.Advance(); var rhs = ParseTerm(state, operators); while (true) { if (state.AtEnd()) break; if (state.Next().Type == TokenType.Operator) { var next_precedence = operators.FindPrecedence(state.Next().Value); if (next_precedence > precedence) rhs = ParseExpression(rhs, state, operators, next_precedence, IsTerminal); else break; } else break; } lhs = new Ast.BinaryOperator(lhs.Source, operators.FindOperator(op.Value).Value, lhs, rhs); } }
private static Ast.If ParseIfStatement( TokenStream Stream, ParseContext Context) { if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "IF") throw new CompileError("[009] Impossible error: If parse entered, no if found.", Stream); var r = new Ast.If(Stream.Next()); Stream.Advance(); //Skip 'if' r.Header = ParseExpression(Stream, Context, TokenType.OpenBrace); if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[00A] Expected {", Stream); r.ThenBlock = ParseBlock(Stream, Context); if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "ELSE") { Stream.Advance(); if (Stream.Next().Type == TokenType.OpenBrace) r.ElseBlock = ParseBlock(Stream, Context); else if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "IF") r.ElseBlock = ParseIfStatement(Stream, Context); } return r; }
internal static Type ParseTypeDeclaration(TokenStream Stream, ParseContext Context) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02C] Expected identifier", Stream); var typename = Stream.Next().Value.ToUpper(); String superTypename; Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", Stream); superTypename = Stream.Next().Value.ToUpper(); Stream.Advance(); } else superTypename = "GENERIC"; var r = new Type(superTypename) { Name = typename }; if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[02D] Expected {", Stream); Stream.Advance(); while (Stream.Next().Type != TokenType.CloseBrace) { if (Stream.Next().Value.ToUpper() == "VAR" || Stream.Next().Value.ToUpper() == "VARIABLE") r.Members.Add(ParseMemberDeclaration(Stream, Context)); else throw new CompileError("[02E] Expected var", Stream); } Stream.Advance(); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; }
private static bool IsEndOfStatement(TokenStream Stream) { if (Stream.AtEnd()) return true; return Stream.Next().Type == TokenType.Semicolon; }
internal static Variable ParseMemberDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next()); Stream.Advance(); var r = new Variable(); r.StorageMethod = VariableStorageMethod.Member; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next()); r.Name = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next()); r.DeclaredTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next()); Stream.Advance(); return r; }
internal static Declaration ParseRuleDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.AtEnd()) throw new CompileError("[02F] Impossible error: ParseRuleDeclaration entered at end of stream.", Stream); try { var r = new Declaration(); r.ReturnTypeName = "RULE-RESULT"; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[030] Expected identifier", Stream.Next()); r.Type = DeclarationType.Rule; Stream.Advance(); r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBraceOrWhen); foreach (var t in r.Terms) if (t.Type == DeclarationTermType.Operator) throw new CompileError("Rule headers cannot contain operators", Stream); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream); r.ReturnTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (Stream.Next().Value.ToUpper() == "WHEN") { Stream.Advance(); r.WhenClause = new WhenClause(ParseExpression(Stream, Context, (stream) => { if (stream.Next().Type == TokenType.OpenBrace) return true; if (stream.Next().Type == TokenType.Identifier && (stream.Next().Value.ToUpper() == "WITH" || stream.Next().Value.ToUpper() == "ORDER")) return true; return false; })); } r.OrderOperator = OrderOperator.NONE; if (Stream.Next().Value.ToUpper() == "ORDER") { Stream.Advance(); var temp = OrderOperator.NONE; if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp)) throw new CompileError("Expected FIRST or LAST", Stream); if (temp != OrderOperator.FIRST && temp != OrderOperator.LAST) throw new CompileError("Expected FIRST or LAST", Stream); Stream.Advance(); r.OrderOperator |= temp; } if (Stream.Next().Value.ToUpper() == "WITH") { Stream.Advance(); var temp = OrderOperator.NONE; if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp)) throw new CompileError("Expected HIGH or LOW", Stream); if (temp != OrderOperator.HIGH && temp != OrderOperator.LOW) throw new CompileError("Expected HIGH or LOW", Stream); Stream.Advance(); if (Stream.Next().Value.ToUpper() != "PRIORITY") throw new CompileError("Expected PRIORITY", Stream); Stream.Advance(); r.OrderOperator |= temp; } if (Stream.Next().Type == TokenType.OpenBrace) { r.Body = new LambdaBlock(ParseBlock(Stream, Context)); } else { throw new CompileError("[031] Expected block", Stream); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; } catch (CompileError ce) { throw ce; } catch (Exception e) { throw new CompileError(e.Message + e.StackTrace, Stream); } }
internal static List<DeclarationTerm> ParseMacroDeclarationHeader(TokenStream Stream, DeclarationHeaderTerminatorType TerminatorType) { var r = new List<DeclarationTerm>(); while (true) { if (TerminatorType == DeclarationHeaderTerminatorType.StreamEnd && Stream.AtEnd()) return r; else if (Stream.Next().Type == TokenType.Colon) return r; else if (TerminatorType == DeclarationHeaderTerminatorType.OpenBrace && Stream.Next().Type == TokenType.OpenBrace) return r; else if (TerminatorType == DeclarationHeaderTerminatorType.OpenBraceOrWhen) { if (Stream.Next().Type == TokenType.OpenBrace) return r; else if (Stream.Next().Value.ToUpper() == "WHEN") return r; else if (Stream.Next().Value.ToUpper() == "WITH") return r; else if (Stream.Next().Value.ToUpper() == "ORDER") return r; } r.Add(ParseDeclarationTerm(Stream)); } }
internal static Declaration ParseMacroDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.AtEnd()) throw new CompileError("[028] Impossible error: ParseDeclaration entered at end of stream.", Stream); try { var r = new Declaration(); r.ReturnTypeName = "VOID"; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[029] Expected identifier", Stream.Next()); r.Type = DeclarationType.Macro; Stream.Advance(); r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBrace); foreach (var t in r.Terms) if (t.Type == DeclarationTermType.Operator) r.DefinesOperator = true; if (r.DefinesOperator) { if (r.Terms.Count != 3 || r.Terms[0].Type != DeclarationTermType.Term || r.Terms[1].Type != DeclarationTermType.Operator || r.Terms[2].Type != DeclarationTermType.Term) throw new CompileError("Operator macros must be of the form 'term op term'", Stream); r.Terms[1].Type = DeclarationTermType.Keyword; } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream); r.ReturnTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (r.DefinesOperator && r.ReturnTypeName == "VOID") throw new CompileError("Operator macros must return a value.", Stream); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.OpenBrace) { r.Body = new LambdaBlock(ParseBlock(Stream, Context)); } else throw new CompileError("[02B] Expected block", Stream); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; } catch (CompileError ce) { throw ce; } catch (Exception e) { throw new CompileError(e.Message + e.StackTrace, Stream); } }
internal static Variable ParseGlobalDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next()); Stream.Advance(); var r = new Variable(); r.StorageMethod = VariableStorageMethod.Member; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next()); var start = Stream.Next(); r.Name = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next()); r.DeclaredTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=") { Stream.Advance(); var initialValue = ParseExpression(Stream, Context, TokenType.Semicolon); var initializer = new Ast.Let(start, new Ast.Identifier(start), initialValue); Context.Initialization.Add(initializer); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next()); Stream.Advance(); return r; }
public static List<Declaration> Build( TokenStream Stream, ParseContext Context, Func<String,ErrorStrategy> OnError, bool PrepareInitializer = true) { var r = new List<Declaration>(); while (!Stream.AtEnd()) { try { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[036] Expected identifier", Stream); if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION") { var declaration = ParseMacroDeclaration(Stream, Context); declaration.OwnerContextID = Context.ID; Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "TEST") { var declaration = ParseMacroDeclaration(Stream, Context); declaration.Type = DeclarationType.Test; declaration.OwnerContextID = Context.ID; r.Add(declaration); Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "RULE") { var declaration = ParseRuleDeclaration(Stream, Context); declaration.OwnerContextID = Context.ID; var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms); if (rulebook == null) { rulebook = new Rulebook { DeclarationTerms = new List<DeclarationTerm>( declaration.Terms.Select(t => t.GenericClone())) }; rulebook.ResultTypeName = declaration.ReturnTypeName; Context.Rules.Rulebooks.Add(rulebook); } //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false) // throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream); if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper()) throw new CompileError("Rule return type not compatible with existing rulebook", Stream); rulebook.Rules.Add(declaration); Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "TYPE") { var type = ParseTypeDeclaration(Stream, Context); if (Context.ActiveScope.FindType(type.Name) != null) throw new CompileError("[038] Type already defined", Stream); Context.ActiveScope.Types.Add(type); } else if (Stream.Next().Value.ToUpper() == "GLOBAL") { var variable = ParseGlobalDeclaration(Stream, Context); variable.StorageMethod = VariableStorageMethod.Static; Context.ActiveScope.Variables.Add(variable); } else if (Stream.Next().Value.ToUpper() == "INCLUDE") { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected string", Stream); var filename = Stream.Next().Value; Stream.Advance(); if (Context.FileLoader == null) throw new CompileError("Inclusion not enabled", Stream); var file = Context.FileLoader.LoadFile(filename, Stream.CurrentFile); var newStream = new TokenStream(new StringIterator(file.Data), Context); newStream.CurrentFile = file; r.AddRange(Build(newStream, Context, OnError, false)); } else if (Stream.Next().Value.ToUpper() == "DEFAULT") { Stream.Advance(); if (Stream.Next().Value.ToUpper() != "OF") throw new CompileError("Expected 'OF'", Stream); Stream.Advance(); if (Stream.Next().Value.ToUpper() != "RULE") throw new CompileError("Expected 'RULE'", Stream); var declaration = ParseRuleDeclaration(Stream, Context); foreach (var term in declaration.Terms) if (!String.IsNullOrEmpty(term.DeclaredTypeName) && term.DeclaredTypeName != "GENERIC") throw new CompileError("Don't declare types for default rules."); declaration.OwnerContextID = Context.ID; var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms); if (rulebook == null) { rulebook = new Rulebook { DeclarationTerms = declaration.Terms }; rulebook.ResultTypeName = declaration.ReturnTypeName; Context.Rules.Rulebooks.Add(rulebook); } //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false) // throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream); if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper()) throw new CompileError("Rule return type not compatible with existing rulebook", Stream); if (rulebook.DefaultValue != null) throw new CompileError("Rulebook already has a default value", Stream); rulebook.DefaultValue = declaration; Context.PendingEmission.Add(declaration); } else throw new CompileError("[039] Unknown declaration type", Stream); } catch (Exception e) { if (OnError(e.Message) == ErrorStrategy.Abort) return r; Stream.Advance(); //Prevent an error from causing an infinite loop } } if (PrepareInitializer) { Context.InitializationFunction = Declaration.Parse("test initialize-globals : void"); var body = new Ast.BlockStatement(new Token()); body.Statements = Context.Initialization; Context.InitializationFunction.Body.Body = body; Context.PendingEmission.Add(Context.InitializationFunction); } return r; }
internal static DeclarationTerm ParseDeclarationTerm(TokenStream Stream) { DeclarationTerm r = null; var start = Stream.Next(); if (Stream.Next().Type == TokenType.Identifier) { r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Keyword, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); } else if (Stream.Next().Type == TokenType.Operator) { r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Operator, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); } else if (Stream.Next().Type == TokenType.OpenParen) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[021] Expected identifier", start); r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Term, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[022] Expected identifier", start); var declaredType = Stream.Next().Value; r.DeclaredTypeName = declaredType.ToUpper(); Stream.Advance(); } if (Stream.Next().Type != TokenType.CloseParen) throw new CompileError("[023] Expected )", start); Stream.Advance(); } else throw new CompileError("[025] Illegal token in declaration header", start); if (!Stream.AtEnd()) { if (Stream.Next().Type == TokenType.QuestionMark) { var marker = Stream.Next(); var repetitionMarker = Stream.Next().Value; Stream.Advance(); if (repetitionMarker == "?") { if (r.Type != DeclarationTermType.Keyword) throw new CompileError("[026] Only keywords can be optional in a declaration header", Stream); r.RepetitionType = DeclarationTermRepetitionType.Optional; } //Left over from when terms could be repeated. //else if (repetitionMarker == "+") // r.RepetitionType = DeclarationTermRepetitionType.OneOrMany; //else if (repetitionMarker == "*") // r.RepetitionType = DeclarationTermRepetitionType.NoneOrMany; else throw new CompileError("[027] Unrecognized repetition marker on declaration term", marker); } } return r; }