public AstNodeArgs(BnfTerm term, CompilerContext context, SourceSpan span, AstNodeList childNodes) { Context = context; Term = term; Span = span; ChildNodes = childNodes; }
public override Token TryMatch(CompilerContext context, ISourceStream source) { bool ignoreCase = !Grammar.CaseSensitive; //Check starting symbol if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null; //Find end symbol source.Position += StartSymbol.Length; while(!source.EOF()) { int firstCharPos; if (EndSymbols.Count == 1) firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position); else firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position); if (firstCharPos < 0) { source.Position = source.Text.Length; if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end. return Token.Create(this, context, source.TokenStart, source.GetLexeme()); else return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"); } //We found a character that might start an end symbol; let's see if it is true. source.Position = firstCharPos; foreach (string endSymbol in EndSymbols) if (source.MatchSymbol(endSymbol, ignoreCase)) { //We found end symbol source.Position += endSymbol.Length; return Token.Create(this, context, source.TokenStart, source.GetLexeme()); }//if source.Position++; //move to the next char and try again }//while return null; //never happens }
public void DoDefaultProcessing(AstNode astRoot, CompilerContext context) { Scope rootScope = new Scope(astRoot, null); astRoot.Scope = rootScope; RunPhases(astRoot, context, AstProcessingPhase.CreatingScopes, AstProcessingPhase.Allocating, AstProcessingPhase.Linking, AstProcessingPhase.MarkTailCalls); }
public void RunPhases(AstNode astRoot, CompilerContext context, params AstProcessingPhase[] phases) { IEnumerable<AstNode> allNodes = astRoot.GetAll(); foreach (AstProcessingPhase phase in phases) { foreach (AstNode node in allNodes) node.OnAstProcessing(context, phase); }//foreach phase }
public override Token TryMatch(CompilerContext context, ISourceStream source) { Match m = _expression.Match(source.Text, source.Position); if (!m.Success) return null; source.Position += m.Length + 1; string text = source.GetLexeme(); return Token.Create(this, context, source.TokenStart, text); }
public override Token TryMatch(CompilerContext context, ISourceStream source) { Token token = base.TryMatch(context, source); if (token != null && (IdentifierPattern.IsMatch(token.Text) && !token.IsKeyword)) { return token; } else { return null; } }
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) { _prevLine = 0; _indents.Clear(); foreach (Token token in tokens) { if (token.Terminal == Grammar.Eof) { yield return CreateSpecialToken(Grammar.NewLine, context, token.Location); //this is necessary, because grammar rules reference newLine terminator //unindent all buffered indents if (_trackIndents) foreach (int i in _indents) yield return CreateSpecialToken(Grammar.Dedent, context, token.Location); _indents.Clear(); //return EOF token yield return token; yield break; }//if Eof //Now deal with normal, non-EOF tokens //We intercept only content tokens on new lines if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) { yield return token; continue; } //if we are here, we have content token on new line; produce newLine token and possibly indents yield return CreateSpecialToken(Grammar.NewLine, context, token.Location); _prevLine = token.Location.Line; if (!_trackIndents) { yield return token; continue; } //Now take care of indents int currIndent = token.Location.Column; int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek(); if (currIndent > prevIndent) { _indents.Push(currIndent); yield return CreateSpecialToken(Grammar.Indent, context, token.Location); } else if (currIndent < prevIndent) { //produce one or more dedent tokens while popping indents from stack while (_indents.Count > 0 && _indents.Peek() > currIndent) { _indents.Pop(); yield return CreateSpecialToken(Grammar.Dedent, context, token.Location); } if (_indents.Count == 0 || _indents.Peek() != currIndent) { yield return Grammar.CreateSyntaxErrorToken (context, token.Location, "Invalid dedent level, no previous matching indent found."); //TODO: add error recovery here } }//else if currIndent < prevIndent yield return token; } //foreach token }
//Used in unit tests public static CompilerContext CreateDummy() { CompilerContext ctx = new CompilerContext(LanguageCompiler.CreateDummy()); return(ctx); }
protected virtual Token QuickParse(CompilerContext context, ISourceStream source) { return(null); }
public static Token Create(Terminal term, CompilerContext context, SourceLocation location, string text) { return(Create(term, context, location, text, text)); }
public virtual string GetSyntaxErrorMessage(CompilerContext context, StringList expectedList) { return(null); //Irony then would construct default message }
// Override this method in language grammar if you want a custom node creation mechanism. public virtual AstNode CreateNode(CompilerContext context, ActionRecord reduceAction, SourceSpan sourceSpan, AstNodeList childNodes) { return null; }
//This method is called if Scanner failed to produce token public virtual Token TryMatch(CompilerContext context, ISourceStream source) { return null; }
public int Priority; //default is 0 #endregion #region virtuals public virtual Token TryMatch(CompilerContext context, ISourceStream source) { return(null); }
public override Token TryMatch(CompilerContext context, ISourceStream source) { return _handler(this, context, source); }
public AstNode Parse(CompilerContext context, SourceFile source) { _context = context; int start = Environment.TickCount; Scanner.Prepare(context, source); IEnumerable<Token> tokenStream = Scanner.BeginScan(); //chain all token filters foreach (TokenFilter filter in Grammar.TokenFilters) { tokenStream = filter.BeginFiltering(context, tokenStream); } //finally, parser takes token stream and produces root Ast node AstNode rootNode = Parser.Parse(context, tokenStream); _compileTime = Environment.TickCount - start; return rootNode; }
}//method public AstNode Parse(CompilerContext context, IEnumerable <Token> tokenStream) { _context = context; _caseSensitive = _context.Compiler.Grammar.CaseSensitive; Reset(); _input = tokenStream.GetEnumerator(); NextToken(); while (true) { if (_currentState == Data.FinalState) { AstNode result = Stack[0].Node; Stack.Reset(); return(result); } //check for scammer error if (_currentToken.Terminal.Category == TokenCategory.Error) { ReportScannerError(); if (!Recover()) { return(null); } continue; } //Get action ActionRecord action = GetCurrentAction(); if (action == null) { ReportParserError(); if (!Recover()) { return(null); //did not recover } continue; }//action==null if (action.HasConflict()) { action = Data.Grammar.OnActionConflict(this, _currentToken, action); } this.OnActionSelected(_currentToken, action); switch (action.ActionType) { case ParserActionType.Operator: if (GetActionTypeForOperation(_currentToken) == ParserActionType.Shift) { goto case ParserActionType.Shift; } else { goto case ParserActionType.Reduce; } case ParserActionType.Shift: ExecuteShiftAction(action); break; case ParserActionType.Reduce: ExecuteReduceAction(action); break; } //switch } //while } //Parse
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9 // so we try to do a quick parse for these, without starting the whole general process protected override Token QuickParse(CompilerContext context, ISourceStream source) { char current = source.CurrentChar; if (char.IsDigit(current) && QuickParseTerminators.IndexOf(source.NextChar) >= 0) { int iValue = current - '0'; object value = null; switch (DefaultIntTypes[0]) { case TypeCode.Int32: value = iValue; break; case TypeCode.UInt32: value = (UInt32)iValue; break; case TypeCode.Byte: value = (byte)iValue; break; case TypeCode.SByte: value = (sbyte) iValue; break; case TypeCode.Int16: value = (Int16)iValue; break; case TypeCode.UInt16: value = (UInt16)iValue; break; default: return null; } Token token = Token.Create(this, context, source.TokenStart, current.ToString(), value); source.Position++; return token; } else return null; }
public void Prepare(CompilerContext context, ISourceStream source) { _context = context; _caseSensitive = context.Compiler.Grammar.CaseSensitive; _source = source; _currentToken = null; _bufferedTokens.Clear(); ResetSource(); }
private Token CreateSpecialToken(Terminal term, CompilerContext context, SourceLocation location) { return Token.Create(term, context, location, string.Empty); }
public virtual IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens) { yield break; }
public virtual string GetSyntaxErrorMessage(CompilerContext context, StringList expectedList) { return null; //Irony then would construct default message }
//Used in unit tests public static CompilerContext CreateDummy() { CompilerContext ctx = new CompilerContext(LanguageCompiler.CreateDummy()); return ctx; }
public static Token CreateSyntaxErrorToken(CompilerContext context, SourceLocation location, string message, params object[] args) { if (args != null && args.Length > 0) message = string.Format(message, args); return Token.Create(Grammar.SyntaxError, context, location, message); }
public override Token TryMatch(CompilerContext context, ISourceStream source) { return(_handler(this, context, source)); }
public AstNode Parse(CompilerContext context, IEnumerable<Token> tokenStream) { _context = context; _caseSensitive = _context.Compiler.Grammar.CaseSensitive; Reset(); _input = tokenStream.GetEnumerator(); NextToken(); while (true) { if (_currentState == Data.FinalState) { AstNode result = Stack[0].Node; Stack.Reset(); return result; } //check for scammer error if (_currentToken.Terminal.Category == TokenCategory.Error) { ReportScannerError(); if (!Recover()) return null; continue; } //Get action ActionRecord action = GetCurrentAction(); if (action == null) { ReportParserError(); if (!Recover()) return null; //did not recover continue; }//action==null if (action.HasConflict()) action = Data.Grammar.OnActionConflict(this, _currentToken, action); this.OnActionSelected(_currentToken, action); switch (action.ActionType) { case ParserActionType.Operator: if (GetActionTypeForOperation(_currentToken) == ParserActionType.Shift) goto case ParserActionType.Shift; else goto case ParserActionType.Reduce; case ParserActionType.Shift: ExecuteShiftAction(action); break; case ParserActionType.Reduce: ExecuteReduceAction(action); break; }//switch }//while }
// Override this method in language grammar if you want a custom node creation mechanism. public virtual AstNode CreateNode(CompilerContext context, ActionRecord reduceAction, SourceSpan sourceSpan, AstNodeList childNodes) { return(null); }