public void AllowsRepeatableTraversalWhileTraversingUnderlyingEnumeratorItemsAtMostOnce() { var tokens = new TokenStream(Tokens()); tokens.Current.ShouldBe(upper, "ABC", 1, 1); tokens.Advance().Current.ShouldBe(lower, "def", 1, 4); tokens.Advance().Advance().Current.ShouldBe(upper, "GHI", 1, 7); tokens.Advance().Advance().Advance().Current.ShouldBe(TokenKind.EndOfInput, "", 1, 10); tokens.Advance().ShouldBeSameAs(tokens.Advance()); }
public void TokenStream_Whitespace() { TokenStream iter = Helpers.MakeTokenStream("* .foo"); Assert.IsFalse(iter.IsWhiteSpaceBeforeCurrentToken()); Assert.IsTrue(iter.IsWhiteSpaceAfterCurrentToken()); iter.Advance(1); Assert.IsTrue(iter.IsWhiteSpaceBeforeCurrentToken()); Assert.IsFalse(iter.IsWhiteSpaceAfterCurrentToken()); iter.Advance(1); Assert.IsFalse(iter.IsWhiteSpaceBeforeCurrentToken()); Assert.IsFalse(iter.IsWhiteSpaceAfterCurrentToken()); }
public void Test_Stream_Advance_type() { List <ScannerToken> tokens = new List <ScannerToken>(); tokens.Add(new ScannerToken(TokenType.PROG, "", 1, 1)); tokens.Add(new ScannerToken(TokenType.VAR, "a", 1, 1)); tokens.Add(new ScannerToken(TokenType.ASSIGN, 1, 2)); tokens.Add(new ScannerToken(TokenType.NUMERIC_INT, "5", 1, 2)); TokenStream streamToken = new TokenStream(tokens); streamToken.Advance(); streamToken.Advance(); Assert.AreEqual(streamToken.Peek().Type, TokenType.NUMERIC_INT); }
public void ProvidesEndOfInputTokenWhenGivenEmptyEnumerator() { var tokens = new TokenStream(Empty()); tokens.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 1); tokens.Advance().ShouldBeSameAs(tokens); }
private ASTNode ParseInternal(string input) { tokenStream = new TokenStream(ExpressionTokenizer.Tokenize(input, StructList <ExpressionToken> .Get())); expressionStack = expressionStack ?? StackPool <ASTNode> .Get(); operatorStack = operatorStack ?? StackPool <OperatorNode> .Get(); if (tokenStream.Current == ExpressionTokenType.ExpressionOpen) { tokenStream.Advance(); } if (!tokenStream.HasMoreTokens) { throw new ParseException("Failed trying to parse empty expression"); } if (tokenStream.Last == ExpressionTokenType.ExpressionClose) { tokenStream.Chop(); } ASTNode retn = ParseLoop(); Release(); return(retn); }
private static AstExpression ParseBinaryOpHelper(TokenStream tks, AstExpression lhs, int minPrec) { var si = tks.SourceInfo; var binOps = BinaryOpPrec.Select(kvp => kvp.Key).ToArray(); while (true) { var op = tks.Current; var thisPrec = GetBinOpPrec(op); if (thisPrec < minPrec) { return(lhs); } tks.ThrowIfTokenIsNotIn(binOps); tks.Advance(); var rhs = ParseUnaryOp(tks); var next = tks.Current; var nextPrec = GetBinOpPrec(next); if (thisPrec < nextPrec || (thisPrec == nextPrec && IsBinOpRightAssoc(op))) { rhs = ParseBinaryOpHelper(tks, rhs, thisPrec + 1); } lhs = new AstBinaryOp(si, lhs, op.Type, rhs); } }
public void ProvidesEndOfInputTokenAfterEnumeratorIsExhausted() { var tokens = new TokenStream(OneToken()); var end = tokens.Advance(); end.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 4); end.Advance().ShouldBeSameAs(end); }
/// <inheritdoc/> public override IReply <TValue> Parse(TokenStream tokens) { if (tokens.Current.Kind == _kind) { return(new Success <TValue>(_value, tokens.Advance())); } return(new Failure <TValue>(tokens, FailureMessage.Expected(_kind.Name))); }
/// <inheritdoc/> public override IGeneralReply ParseGenerally(TokenStream tokens) { if (tokens.Current.Kind == _kind) { return(new GeneralSuccess(tokens.Advance())); } return(new GeneralFailure(tokens, FailureMessage.Expected(_kind.Name))); }
public Reply <Token> Parse(TokenStream tokens) { if (tokens.Current.Literal == expectation) { return(new Parsed <Token>(tokens.Current, tokens.Advance())); } return(new Error <Token>(tokens, ErrorMessage.Expected(expectation))); }
public Reply <Token> Parse(TokenStream tokens) { if (tokens.Current.Kind == kind) { return(new Parsed <Token>(tokens.Current, tokens.Advance())); } return(new Error <Token>(tokens, ErrorMessage.Expected(kind.Name))); }
public void ParseTest() { string[] samples1 = new string[] { "100px", "200%", "-3grad", "0.8em", "-0.5deg", "22s", "345.7Hz", "144dpi", "12dppx", "12st", "0.7vmin", "0vmax", "1.117gr", "-6.0dB", "+12dB", }; UnitType[] types1 = new UnitType[] { UnitType.Length, UnitType.Percentage, UnitType.Angle, UnitType.Length, UnitType.Angle, UnitType.Time, UnitType.Frequency, UnitType.Resolution, UnitType.Resolution, UnitType.Semitones, UnitType.Viewport, UnitType.Viewport, UnitType.Grid, UnitType.Volume, UnitType.Volume, }; int i = 0; foreach (string text in samples1) { ITextProvider tp = new StringTextProvider(text); TokenStream tokens = Helpers.MakeTokenStream(tp); tokens.Advance(1); UnitValue uv = new UnitValue(); uv.Parse(new ItemFactory(tp, null), tp, tokens); Assert.AreEqual(types1[i++], uv.UnitType); Assert.AreEqual(CssTokenType.Units, uv.UnitToken.TokenType); } }
/// <inheritdoc/> public override IReply <string> Parse(TokenStream tokens) { var currentToken = tokens.Current; if (currentToken.Kind != _kind) { return(new Failure <string>(tokens, FailureMessage.Expected(_kind.Name))); } return(new Success <string>(currentToken.Lexeme, tokens.Advance())); }
/// <inheritdoc/> public override IReply <TValue> Parse(TokenStream tokens) { if (tokens.Current.Kind != _kind) { return(new Failure <TValue>(tokens, FailureMessage.Expected(_kind.Name))); } var parsedValue = _lexemeMapping(tokens.Current.Lexeme); return(new Success <TValue>(parsedValue, tokens.Advance())); }
public void TokenStream_Simple() { TokenStream iter = Helpers.MakeTokenStream(".foo"); Assert.AreEqual(CssTokenType.Dot, iter.CurrentToken.TokenType); Assert.AreEqual(CssTokenType.Identifier, iter.Peek(1).TokenType); Assert.AreEqual(CssTokenType.EndOfFile, iter.Peek(2).TokenType); Assert.AreEqual(0, iter.Position); Assert.AreEqual(3, iter.Tokens.Count); Assert.AreEqual(iter.Tokens[0], iter.Advance(1)); Assert.AreEqual(1, iter.Position); Assert.AreEqual(CssTokenType.Identifier, iter.CurrentToken.TokenType); iter.Advance(-1); Assert.AreEqual(0, iter.Position); iter.Advance(2); Assert.AreEqual(CssTokenType.EndOfFile, iter.CurrentToken.TokenType); }
public void HtmlComment_ParseTest() { string text1 = "<!-- foo#bar -->"; ITextProvider tp = new StringTextProvider(text1); TokenStream tokens = Helpers.MakeTokenStream(tp); // Parse "<!--" { HtmlComment hc = new HtmlComment(); Assert.IsTrue(hc.Parse(new ItemFactory(tp, null), tp, tokens)); Assert.AreEqual(1, hc.Children.Count); Assert.AreEqual(CssTokenType.OpenHtmlComment, ((TokenItem)hc.Children[0]).TokenType); } // Parse "foo#bar" { Assert.AreEqual(CssTokenType.Identifier, tokens.Advance(1).TokenType); Assert.AreEqual(CssTokenType.HashName, tokens.Advance(1).TokenType); } // Parse "-->" { HtmlComment hc = new HtmlComment(); Assert.IsTrue(hc.Parse(new ItemFactory(tp, null), tp, tokens)); Assert.AreEqual(1, hc.Children.Count); Assert.AreEqual(CssTokenType.CloseHtmlComment, ((TokenItem)hc.Children[0]).TokenType); } string text2 = "<!-- @namespace foo \"www.foo.com\" -->"; CssParser parser = new CssParser(); StyleSheet sheet = parser.Parse(text2, true); Assert.IsTrue(sheet.ComplexItemFromRange(14, 0) is NamespaceDirective); Assert.IsTrue(sheet.ComplexItemFromRange(2, 0) is HtmlComment); Assert.IsTrue(sheet.ComplexItemFromRange(35, 0) is HtmlComment); }
private static IEnumerable <IArgumentInfo> ParseArgumentItem(RdParseContext context) { List <IArgumentInfo> arguments = null; TokenStream <RdToken> tokens = context.Tokens; tokens.Advance(1); // Past '\item'. Inside { } we can find any number of '\dots' which are keywords. Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace); if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace) { int startTokenIndex, endTokenIndex; if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex)) { TextRange range = TextRange.FromBounds(tokens[startTokenIndex].End, tokens[endTokenIndex].Start); string argumentsText = context.TextProvider.GetText(range); string[] argumentNames = argumentsText.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); arguments = new List <IArgumentInfo>(); // Move past \item{} tokens.Position = endTokenIndex + 1; Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace); if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace) { string description = RdText.GetText(context); foreach (string n in argumentNames) { string name = n.Trim(); if (name == @"\dots") { name = "..."; } ArgumentInfo info = new ArgumentInfo(name, description.Trim()); arguments.Add(info); } } } } return(arguments); }
// <package-ident> ::= <ident> ( '.' <ident> )* private static AstPackageIdent ParsePackageIdent(TokenStream tks) { var si = tks.SourceInfo; var idents = new List <string>(); while (true) { var ident = tks.Expect(TokenType.Ident).StringValue; idents.Add(ident); if (tks.Peek().Type == TokenType.Dot && tks.Peek(1).Type == TokenType.Ident) { tks.Advance(); continue; } break; } return(new AstPackageIdent(si, idents)); }
public void ProvidesPositionOfCurrentToken() { var tokens = new TokenStream(Tokens()); tokens.Position.Line.ShouldBe(1); tokens.Position.Column.ShouldBe(1); tokens.Advance().Position.Line.ShouldBe(1); tokens.Advance().Position.Column.ShouldBe(4); tokens.Advance().Advance().Position.Line.ShouldBe(1); tokens.Advance().Advance().Position.Column.ShouldBe(7); tokens.Advance().Advance().Advance().Position.Line.ShouldBe(1); tokens.Advance().Advance().Advance().Position.Column.ShouldBe(10); }
public static Ast.ComplexString ParseComplexString(TokenStream Stream, ParseContext Context) { System.Diagnostics.Debug.Assert(Stream.Next().Type == TokenType.Dollar); var start = Stream.Next(); Stream.PushState(TokenStreamState.ComplexString); //Enter complex string parsing mode Stream.Advance(); if (Stream.Next().Type != TokenType.ComplexStringQuote) throw new CompileError("[015] Expected \"", Stream); Stream.Advance(); var stringPieces = new List<Ast.Node>(); while (Stream.Next().Type != TokenType.ComplexStringQuote) { if (Stream.Next().Type == TokenType.ComplexStringPart) { stringPieces.Add(new Ast.StringLiteral(Stream.Next(), Stream.Next().Value)); Stream.Advance(); } else if (Stream.Next().Type == TokenType.OpenBracket) { Stream.PushState(TokenStreamState.Normal); //Make sure we are parsing normally for the Stream.Advance(); //Skip the [ //embedded expression stringPieces.Add(ParseExpression(Stream, Context, TokenType.CloseBracket)); if (Stream.Next().Type != TokenType.CloseBracket) //Shouldn't be possible throw new CompileError("[016] Expected ]", Stream); Stream.PopState(); //Return to complex string parsing mode Stream.Advance(); } else throw new InvalidProgramException(); } Stream.PopState(); //Return to normal parsing mode Stream.Advance(); return new Ast.ComplexString(start, stringPieces); }
public static List<Ast.Node> ParseStaticInvokation( TokenStream Stream, ParseContext Context) { if (Stream.Next().Type != TokenType.OpenBracket) throw new CompileError("[017] Expected [", Stream); Stream.Advance(); var parameters = new List<Ast.Node>(); while (true) { if (IsEndOfStatement(Stream)) throw new CompileError("[018] Expected ]", Stream); if (Stream.Next().Type == TokenType.CloseBracket) { Stream.Advance(); return parameters; } parameters.Add(ParseTerm(Stream, Context)); } }
public static Declaration Parse(String Header) { var tokenIterator = new TokenStream(new StringIterator(Header), new ParseContext()); var headerTerms = EtcScriptLib.Parse.ParseMacroDeclarationHeader(tokenIterator, EtcScriptLib.Parse.DeclarationHeaderTerminatorType.StreamEnd); var r = new Declaration(); r.ReturnTypeName = "VOID"; if (!tokenIterator.AtEnd() && tokenIterator.Next().Type == TokenType.Colon) { tokenIterator.Advance(); if (tokenIterator.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", tokenIterator); r.ReturnTypeName = tokenIterator.Next().Value.ToUpper(); tokenIterator.Advance(); } if (!tokenIterator.AtEnd()) throw new CompileError("Header did not end when expected"); r.Terms = headerTerms; r.Body = new LambdaBlock(null); return r; }
private static Ast.Statement ParseStatement( TokenStream Stream, ParseContext Context) { Ast.Statement r = null; var firstToken = Stream.Next().Value.ToUpper(); if (firstToken == "LET") { r = ParseLetStatement(Stream, Context); } else if (firstToken == "IF") { r = ParseIfStatement(Stream, Context); } else if (firstToken == "RETURN") { r = ParseReturnStatement(Stream, Context); } else if (firstToken == "VAR" || firstToken == "VARIABLE") { r = ParseLocalDeclaration(Stream, Context); } else { //If it's not any special statement, it must be a function call. var parameters = ParseStaticInvokationStatement(Stream, Context); var control = Context.FindControl(parameters); if (control != null) { Ast.Node childBlock = null; if (control.BlockType == ControlBlockType.RequiredBlock) { if (Stream.Next().Type == TokenType.OpenBrace) childBlock = ParseBlock(Stream, Context); else throw new CompileError("[00C] Expected {", Stream); } else { if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00D] Expected ;", Stream); Stream.Advance(); } r = new Ast.ControlInvokation(parameters[0].Source, control, parameters, childBlock); } else { r = new Ast.StaticInvokation(parameters[0].Source, parameters); if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00E] Expected ;", Stream); Stream.Advance(); } } return r; }
internal static Variable ParseMemberDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next()); Stream.Advance(); var r = new Variable(); r.StorageMethod = VariableStorageMethod.Member; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next()); r.Name = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next()); r.DeclaredTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next()); Stream.Advance(); return r; }
internal static Declaration ParseRuleDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.AtEnd()) throw new CompileError("[02F] Impossible error: ParseRuleDeclaration entered at end of stream.", Stream); try { var r = new Declaration(); r.ReturnTypeName = "RULE-RESULT"; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[030] Expected identifier", Stream.Next()); r.Type = DeclarationType.Rule; Stream.Advance(); r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBraceOrWhen); foreach (var t in r.Terms) if (t.Type == DeclarationTermType.Operator) throw new CompileError("Rule headers cannot contain operators", Stream); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream); r.ReturnTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (Stream.Next().Value.ToUpper() == "WHEN") { Stream.Advance(); r.WhenClause = new WhenClause(ParseExpression(Stream, Context, (stream) => { if (stream.Next().Type == TokenType.OpenBrace) return true; if (stream.Next().Type == TokenType.Identifier && (stream.Next().Value.ToUpper() == "WITH" || stream.Next().Value.ToUpper() == "ORDER")) return true; return false; })); } r.OrderOperator = OrderOperator.NONE; if (Stream.Next().Value.ToUpper() == "ORDER") { Stream.Advance(); var temp = OrderOperator.NONE; if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp)) throw new CompileError("Expected FIRST or LAST", Stream); if (temp != OrderOperator.FIRST && temp != OrderOperator.LAST) throw new CompileError("Expected FIRST or LAST", Stream); Stream.Advance(); r.OrderOperator |= temp; } if (Stream.Next().Value.ToUpper() == "WITH") { Stream.Advance(); var temp = OrderOperator.NONE; if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp)) throw new CompileError("Expected HIGH or LOW", Stream); if (temp != OrderOperator.HIGH && temp != OrderOperator.LOW) throw new CompileError("Expected HIGH or LOW", Stream); Stream.Advance(); if (Stream.Next().Value.ToUpper() != "PRIORITY") throw new CompileError("Expected PRIORITY", Stream); Stream.Advance(); r.OrderOperator |= temp; } if (Stream.Next().Type == TokenType.OpenBrace) { r.Body = new LambdaBlock(ParseBlock(Stream, Context)); } else { throw new CompileError("[031] Expected block", Stream); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; } catch (CompileError ce) { throw ce; } catch (Exception e) { throw new CompileError(e.Message + e.StackTrace, Stream); } }
private static Ast.If ParseIfStatement( TokenStream Stream, ParseContext Context) { if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "IF") throw new CompileError("[009] Impossible error: If parse entered, no if found.", Stream); var r = new Ast.If(Stream.Next()); Stream.Advance(); //Skip 'if' r.Header = ParseExpression(Stream, Context, TokenType.OpenBrace); if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[00A] Expected {", Stream); r.ThenBlock = ParseBlock(Stream, Context); if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "ELSE") { Stream.Advance(); if (Stream.Next().Type == TokenType.OpenBrace) r.ElseBlock = ParseBlock(Stream, Context); else if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "IF") r.ElseBlock = ParseIfStatement(Stream, Context); } return r; }
private static List<Ast.Initializer> ParseInitializers( TokenStream Stream, ParseContext Context) { Stream.Advance(); var r = new List<Ast.Initializer>(); while (Stream.Next().Type != TokenType.CloseBrace) { var initializerStart = Stream.Next(); if (Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "LET") throw new CompileError("[01E] Expected let", Stream); Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[01E] Expected identifier", Stream); var memberName = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type != TokenType.Operator && Stream.Next().Value != "=") throw new CompileError("[01F] Expected =", Stream); Stream.Advance(); var value = ParseExpression(Stream, Context, TokenType.Semicolon); if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[020] Expected ;", Stream); Stream.Advance(); r.Add(new Ast.Initializer(initializerStart, memberName, value)); } Stream.Advance(); return r; }
private static Ast.Let ParseLetStatement( TokenStream Stream, ParseContext Context) { var start = Stream.Next(); if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "LET") throw new CompileError("[000] Impossible error: Let parse entered, no let found.", Stream); Stream.Advance(); var LHS = ParseTerm(Stream, Context); if (Stream.Next().Type != TokenType.Operator || Stream.Next().Value != "=") throw new CompileError("[001] Expected '='", Stream); Stream.Advance(); var RHS = ParseExpression(Stream, Context, TokenType.Semicolon); if (!IsEndOfStatement(Stream)) throw new CompileError("[002] Expected ;", Stream); Stream.Advance(); return new Ast.Let(start, LHS, RHS); }
private static Ast.LocalDeclaration ParseLocalDeclaration( TokenStream Stream, ParseContext Context) { var start = Stream.Next(); Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[003] Expected identifier", Stream); var r = new Ast.LocalDeclaration(start); r.Name = Stream.Next().Value; //r.Typename = "GENERIC"; Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[004] Expected identifier", Stream); r.Typename = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=") { Stream.Advance(); r.Value = ParseExpression(Stream, Context, TokenType.Semicolon); } if (!IsEndOfStatement(Stream)) throw new CompileError("[005] Expected ;", Stream); Stream.Advance(); return r; }
private static Ast.Node ParseTerm( TokenStream Stream, ParseContext Context) { if (IsEndOfStatement(Stream)) throw new CompileError("[012] Expected argument", Stream); Ast.Node r = null; if (Stream.Next().Type == TokenType.OpenParen) { Stream.Advance(); r = ParseExpression(Stream, Context, TokenType.CloseParen); Stream.Advance(); //Skip close paren } else if (Stream.Next().Type == TokenType.OpenBracket) { var parameters = ParseStaticInvokation(Stream, Context); r = new Ast.StaticInvokation(parameters[0].Source, parameters); } else if (Stream.Next().Type == TokenType.Identifier || Stream.Next().Type == TokenType.Number || Stream.Next().Type == TokenType.String) { r = new Ast.Identifier(Stream.Next()); Stream.Advance(); } else if (Stream.Next().Type == TokenType.Operator) { r = new Ast.Identifier(Stream.Next()); Stream.Advance(); } else if (Stream.Next().Type == TokenType.Dollar) { return ParseComplexString(Stream, Context); } else if (Stream.Next().Type == TokenType.OpenBrace) { var start = Stream.Next(); Stream.Advance(); var list = new List<Ast.Node>(); while (Stream.Next().Type != TokenType.CloseBrace) list.Add(ParseTerm(Stream, Context)); Stream.Advance(); return new Ast.AssembleList(start, list); } else throw new CompileError("[014] Illegal token in argument list", Stream.Next()); r = ParseOptionalDot(Stream, r, Context); return r; }
private static Ast.Node ParseOptionalDot( TokenStream Stream, Ast.Node LHS, ParseContext Context) { if (Stream.AtEnd()) return LHS; if (Stream.Next().Type == TokenType.Dot) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[00F] Dot operator must be followed by identifier", Stream); var RHS = Stream.Next().Value; var MA = new Ast.MemberAccess(LHS.Source); MA.Object = LHS; MA.Name = RHS.ToUpper(); Stream.Advance(); return ParseOptionalDot(Stream, MA, Context); } else if (Stream.Next().Type == TokenType.At) { Stream.Advance(); var RHS = ParseTerm(Stream, Context); var I = new Ast.Indexer(LHS.Source); I.Object = LHS; I.Index = RHS; return ParseOptionalDot(Stream, I, Context); } else if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[013] Expected identifier", Stream); var C = new Ast.Cast(Stream.Next(), LHS, Stream.Next().Value.ToUpper()); Stream.Advance(); return ParseOptionalDot(Stream, C, Context); } else return LHS; }
//Implements http://en.wikipedia.org/wiki/Operator-precedence_parser private static Ast.Node ParseExpression( Ast.Node lhs, TokenStream state, ParseContext operators, int minimum_precedence, Predicate<TokenStream> IsTerminal) { while (true) { if (IsTerminal(state)) return lhs; if (state.Next().Type != TokenType.Operator) throw new CompileError("[01C] Expected operator", state.Next()); var precedence = operators.FindPrecedence(state.Next().Value); if (precedence < minimum_precedence) return lhs; var op = state.Next(); state.Advance(); var rhs = ParseTerm(state, operators); while (true) { if (state.AtEnd()) break; if (state.Next().Type == TokenType.Operator) { var next_precedence = operators.FindPrecedence(state.Next().Value); if (next_precedence > precedence) rhs = ParseExpression(rhs, state, operators, next_precedence, IsTerminal); else break; } else break; } lhs = new Ast.BinaryOperator(lhs.Source, operators.FindOperator(op.Value).Value, lhs, rhs); } }
internal static Type ParseTypeDeclaration(TokenStream Stream, ParseContext Context) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02C] Expected identifier", Stream); var typename = Stream.Next().Value.ToUpper(); String superTypename; Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", Stream); superTypename = Stream.Next().Value.ToUpper(); Stream.Advance(); } else superTypename = "GENERIC"; var r = new Type(superTypename) { Name = typename }; if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[02D] Expected {", Stream); Stream.Advance(); while (Stream.Next().Type != TokenType.CloseBrace) { if (Stream.Next().Value.ToUpper() == "VAR" || Stream.Next().Value.ToUpper() == "VARIABLE") r.Members.Add(ParseMemberDeclaration(Stream, Context)); else throw new CompileError("[02E] Expected var", Stream); } Stream.Advance(); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; }
public void TryingToAdvanceBeyondEndOfInputResultsInNoMovement() { var tokens = new TokenStream(Empty()); tokens.ShouldBeSameAs(tokens.Advance()); }
internal static Declaration ParseMacroDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.AtEnd()) throw new CompileError("[028] Impossible error: ParseDeclaration entered at end of stream.", Stream); try { var r = new Declaration(); r.ReturnTypeName = "VOID"; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[029] Expected identifier", Stream.Next()); r.Type = DeclarationType.Macro; Stream.Advance(); r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBrace); foreach (var t in r.Terms) if (t.Type == DeclarationTermType.Operator) r.DefinesOperator = true; if (r.DefinesOperator) { if (r.Terms.Count != 3 || r.Terms[0].Type != DeclarationTermType.Term || r.Terms[1].Type != DeclarationTermType.Operator || r.Terms[2].Type != DeclarationTermType.Term) throw new CompileError("Operator macros must be of the form 'term op term'", Stream); r.Terms[1].Type = DeclarationTermType.Keyword; } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream); r.ReturnTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (r.DefinesOperator && r.ReturnTypeName == "VOID") throw new CompileError("Operator macros must return a value.", Stream); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.OpenBrace) { r.Body = new LambdaBlock(ParseBlock(Stream, Context)); } else throw new CompileError("[02B] Expected block", Stream); if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } return r; } catch (CompileError ce) { throw ce; } catch (Exception e) { throw new CompileError(e.Message + e.StackTrace, Stream); } }
// consider replacing with access expression since this will always be a root property access private bool ParseIdentifier(ref ASTNode node) { if (tokenStream.Current != ExpressionTokenType.Identifier) { return(false); } node = ASTNode.IdentifierNode(tokenStream.Current); tokenStream.Advance(); return(true); }
internal static Variable ParseGlobalDeclaration(TokenStream Stream, ParseContext Context) { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next()); Stream.Advance(); var r = new Variable(); r.StorageMethod = VariableStorageMethod.Member; if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next()); var start = Stream.Next(); r.Name = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next()); r.DeclaredTypeName = Stream.Next().Value.ToUpper(); Stream.Advance(); } if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=") { Stream.Advance(); var initialValue = ParseExpression(Stream, Context, TokenType.Semicolon); var initializer = new Ast.Let(start, new Ast.Identifier(start), initialValue); Context.Initialization.Add(initializer); } if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark) { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream); r.Documentation = Stream.Next().Value; Stream.Advance(); } if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next()); Stream.Advance(); return r; }
private static Ast.BlockStatement ParseBlock( TokenStream Stream, ParseContext Context) { var r = new Ast.BlockStatement(Stream.Next()); Stream.Advance(); //Skip the opening brace. while (Stream.Next().Type != TokenType.CloseBrace) r.Statements.Add(ParseStatement(Stream, Context)); Stream.Advance(); //Skip the closing brace. return r; }
private static Ast.Return ParseReturnStatement( TokenStream Stream, ParseContext Context) { var start = Stream.Next(); if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "RETURN") throw new CompileError("[007] Impossible error: Return parse entered, no return found.", Stream); Stream.Advance(); var r = new Ast.Return(start); if (Stream.Next().Type != TokenType.Semicolon) r.Value = ParseExpression(Stream, Context, TokenType.Semicolon); if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[008] Expected ;", Stream); Stream.Advance(); return r; }
public void Style(TokenStream Stream) { this.Stream = Stream; this.FoldLevel = 1; while (!Stream.AtEnd()) { if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION" || Stream.Next().Value.ToUpper() == "TEST") ParseMacroDeclaration(); else if (Stream.Next().Value.ToUpper() == "RULE") ParseRuleDeclaration(); else if (Stream.Next().Value.ToUpper() == "TYPE") ParseTypeDeclaration(); else if (Stream.Next().Value.ToUpper() == "GLOBAL") ParseGlobalDeclaration(); else if (Stream.Next().Value.ToUpper() == "INCLUDE") { Style(TokenStyle.Keyword); Expect(TokenType.String); } else if (Stream.Next().Value.ToUpper() == "DEFAULT") { Style(TokenStyle.Keyword); Expect(TokenType.Identifier, t => t.Value.ToUpper() == "OF"); ParseRuleDeclaration(); } else Stream.Advance(); } }
private bool ParseDeclaration(ref ASTNode node) { AttributeNode attrNode = null; LightList <AttributeNode> attributes = LightList <AttributeNode> .Get(); while (ParseAttribute(ref attrNode)) { attributes.Add(attrNode); if (tokenStream.Current != ExpressionTokenType.ArrayAccessOpen) { break; } } if (attributes.size == 0) { LightList <AttributeNode> .Release(ref attributes); } if (tokenStream.Current != ExpressionTokenType.Identifier) { return(false); } // modifiers? -> returnType -> name -> signature -> openBrace * closeBrace tokenStream.Save(); bool isStatic = false; if (tokenStream.Current == "static") { isStatic = true; tokenStream.Advance(); } ExpressionParser parser = new ExpressionParser(tokenStream); StructList <LambdaArgument> signature = null; TypeLookup typeLookup = default; if (!parser.ParseTypePath(ref typeLookup)) { goto fail; } tokenStream.Set(parser.GetTokenPosition()); parser.Release(false); if (tokenStream.Current != ExpressionTokenType.Identifier) { goto fail; } string name = tokenStream.Current.value; tokenStream.Advance(); // if semi colon then we have a field! if (tokenStream.Current == ExpressionTokenType.SemiColon) { tokenStream.Advance(); node = new FieldNode() { name = name, isStatic = isStatic, attributes = attributes, typeLookup = typeLookup }; return(true); } if (tokenStream.Current != ExpressionTokenType.ParenOpen) { goto fail; } signature = StructList <LambdaArgument> .Get(); if (tokenStream.NextTokenIs(ExpressionTokenType.ParenClose)) { tokenStream.Advance(2); } else { int matchingIndex = tokenStream.FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose); if (matchingIndex == -1) { goto fail; } TokenStream subStream = tokenStream.AdvanceAndReturnSubStream(matchingIndex); subStream.Advance(); tokenStream.Advance(); if (!ExpressionParser.ParseSignature(subStream, signature)) { goto fail; } for (int i = 0; i < signature.size; i++) { if (signature.array[i].type == null) { throw new ParseException($"When defining a method you must specify a type for all arguments. Found identifier {signature.array[i].identifier} but no type was given."); } } } if (tokenStream.Current != ExpressionTokenType.ExpressionOpen) { goto fail; } BlockNode block = ParseBlock(); node = new MethodNode() { body = block, returnTypeLookup = typeLookup, attributes = attributes, name = name, isStatic = isStatic, signatureList = signature != null?signature.ToArray() : s_EmptySignature }; StructList <LambdaArgument> .Release(ref signature); parser.Release(false); return(true); fail: { tokenStream.Restore(); parser.Release(false); typeLookup.Release(); signature?.Release(); return(false); } }
public static List<Declaration> Build( TokenStream Stream, ParseContext Context, Func<String,ErrorStrategy> OnError, bool PrepareInitializer = true) { var r = new List<Declaration>(); while (!Stream.AtEnd()) { try { if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[036] Expected identifier", Stream); if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION") { var declaration = ParseMacroDeclaration(Stream, Context); declaration.OwnerContextID = Context.ID; Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "TEST") { var declaration = ParseMacroDeclaration(Stream, Context); declaration.Type = DeclarationType.Test; declaration.OwnerContextID = Context.ID; r.Add(declaration); Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "RULE") { var declaration = ParseRuleDeclaration(Stream, Context); declaration.OwnerContextID = Context.ID; var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms); if (rulebook == null) { rulebook = new Rulebook { DeclarationTerms = new List<DeclarationTerm>( declaration.Terms.Select(t => t.GenericClone())) }; rulebook.ResultTypeName = declaration.ReturnTypeName; Context.Rules.Rulebooks.Add(rulebook); } //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false) // throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream); if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper()) throw new CompileError("Rule return type not compatible with existing rulebook", Stream); rulebook.Rules.Add(declaration); Context.PendingEmission.Add(declaration); } else if (Stream.Next().Value.ToUpper() == "TYPE") { var type = ParseTypeDeclaration(Stream, Context); if (Context.ActiveScope.FindType(type.Name) != null) throw new CompileError("[038] Type already defined", Stream); Context.ActiveScope.Types.Add(type); } else if (Stream.Next().Value.ToUpper() == "GLOBAL") { var variable = ParseGlobalDeclaration(Stream, Context); variable.StorageMethod = VariableStorageMethod.Static; Context.ActiveScope.Variables.Add(variable); } else if (Stream.Next().Value.ToUpper() == "INCLUDE") { Stream.Advance(); if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected string", Stream); var filename = Stream.Next().Value; Stream.Advance(); if (Context.FileLoader == null) throw new CompileError("Inclusion not enabled", Stream); var file = Context.FileLoader.LoadFile(filename, Stream.CurrentFile); var newStream = new TokenStream(new StringIterator(file.Data), Context); newStream.CurrentFile = file; r.AddRange(Build(newStream, Context, OnError, false)); } else if (Stream.Next().Value.ToUpper() == "DEFAULT") { Stream.Advance(); if (Stream.Next().Value.ToUpper() != "OF") throw new CompileError("Expected 'OF'", Stream); Stream.Advance(); if (Stream.Next().Value.ToUpper() != "RULE") throw new CompileError("Expected 'RULE'", Stream); var declaration = ParseRuleDeclaration(Stream, Context); foreach (var term in declaration.Terms) if (!String.IsNullOrEmpty(term.DeclaredTypeName) && term.DeclaredTypeName != "GENERIC") throw new CompileError("Don't declare types for default rules."); declaration.OwnerContextID = Context.ID; var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms); if (rulebook == null) { rulebook = new Rulebook { DeclarationTerms = declaration.Terms }; rulebook.ResultTypeName = declaration.ReturnTypeName; Context.Rules.Rulebooks.Add(rulebook); } //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false) // throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream); if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper()) throw new CompileError("Rule return type not compatible with existing rulebook", Stream); if (rulebook.DefaultValue != null) throw new CompileError("Rulebook already has a default value", Stream); rulebook.DefaultValue = declaration; Context.PendingEmission.Add(declaration); } else throw new CompileError("[039] Unknown declaration type", Stream); } catch (Exception e) { if (OnError(e.Message) == ErrorStrategy.Abort) return r; Stream.Advance(); //Prevent an error from causing an infinite loop } } if (PrepareInitializer) { Context.InitializationFunction = Declaration.Parse("test initialize-globals : void"); var body = new Ast.BlockStatement(new Token()); body.Statements = Context.Initialization; Context.InitializationFunction.Body.Body = body; Context.PendingEmission.Add(Context.InitializationFunction); } return r; }
private bool ParseAttribute(ref AttributeNode node) { if (tokenStream.Current != ExpressionTokenType.ArrayAccessOpen) { return(false); } tokenStream.Save(); tokenStream.Advance(); TypeLookup typeLookup = default; ExpressionParser parser = new ExpressionParser(tokenStream); if (!parser.ParseTypePath(ref typeLookup)) { goto fail; } tokenStream.Set(parser.GetTokenPosition()); parser.Release(false); if (tokenStream.Current == ExpressionTokenType.ArrayAccessClose) { tokenStream.Advance(); node = new AttributeNode() { typeLookup = typeLookup }; return(true); } fail: { typeLookup.Release(); parser.Release(false); return(false); } }
internal static DeclarationTerm ParseDeclarationTerm(TokenStream Stream) { DeclarationTerm r = null; var start = Stream.Next(); if (Stream.Next().Type == TokenType.Identifier) { r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Keyword, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); } else if (Stream.Next().Type == TokenType.Operator) { r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Operator, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); } else if (Stream.Next().Type == TokenType.OpenParen) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[021] Expected identifier", start); r = new DeclarationTerm { Name = Stream.Next().Value.ToUpper(), Type = DeclarationTermType.Term, RepetitionType = DeclarationTermRepetitionType.Once }; Stream.Advance(); if (Stream.Next().Type == TokenType.Colon) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[022] Expected identifier", start); var declaredType = Stream.Next().Value; r.DeclaredTypeName = declaredType.ToUpper(); Stream.Advance(); } if (Stream.Next().Type != TokenType.CloseParen) throw new CompileError("[023] Expected )", start); Stream.Advance(); } else throw new CompileError("[025] Illegal token in declaration header", start); if (!Stream.AtEnd()) { if (Stream.Next().Type == TokenType.QuestionMark) { var marker = Stream.Next(); var repetitionMarker = Stream.Next().Value; Stream.Advance(); if (repetitionMarker == "?") { if (r.Type != DeclarationTermType.Keyword) throw new CompileError("[026] Only keywords can be optional in a declaration header", Stream); r.RepetitionType = DeclarationTermRepetitionType.Optional; } //Left over from when terms could be repeated. //else if (repetitionMarker == "+") // r.RepetitionType = DeclarationTermRepetitionType.OneOrMany; //else if (repetitionMarker == "*") // r.RepetitionType = DeclarationTermRepetitionType.NoneOrMany; else throw new CompileError("[027] Unrecognized repetition marker on declaration term", marker); } } return r; }
public void AdvancesToTheNextToken() { var tokens = new TokenStream(Tokens()); tokens.Advance().Current.ShouldBe(lower, "def", 1, 4); }
private static Ast.New ParseNew( TokenStream Stream, ParseContext Context) { Stream.Advance(); if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[01D] Expected identifier", Stream); var r = new Ast.New(Stream.Next()); r.Typename = Stream.Next().Value.ToUpper(); Stream.Advance(); if (Stream.Next().Type == TokenType.OpenBrace) r.Initializers = ParseInitializers(Stream, Context); return r; }