public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) { foreach (Token token in tokens) { if (!token.Terminal.IsSet(TermOptions.IsBrace)) { yield return token; continue; } //open brace symbol if (token.Terminal.IsSet(TermOptions.IsOpenBrace)) { _braces.Push(token); yield return token; continue; } //We have closing brace if (_braces.Count == 0) { yield return context.CreateErrorTokenAndReportError( token.Location, token.Text, "Unmatched closing brace '{0}'", token.Text); continue; } //check match Token last = _braces.Pop(); if (last.AsSymbol.IsPairFor != token.AsSymbol) { yield return context.CreateErrorTokenAndReportError(token.Location, token.Text, "Unmatched closing brace '{0}' - expected '{1}'", last.AsSymbol.IsPairFor.Name); continue; } //everything is ok, there's matching brace on top of the stack Token.LinkMatchingBraces(last, token); context.CurrentParseTree.OpenBraces.Add(last); yield return token; //return this token }//foreach token yield break; }//method
public override Token TryMatch(CompilerContext context, ISourceStream source) { Match m = _expression.Match(source.Text, source.Position); if (!m.Success || m.Index != source.Position) return null; source.Position += m.Length; string text = source.GetLexeme(); return new Token(this, source.TokenStart, text, null); }
public ParseTree Parse(CompilerContext context, string sourceText, string fileName) { context.CurrentParseTree = new ParseTree(sourceText, fileName); Scanner.SetSource(sourceText); Scanner.BeginScan(context); CoreParser.Parse(context); if (context.CurrentParseTree.Errors.Count > 0) context.CurrentParseTree.Errors.Sort(SyntaxErrorList.ByLocation); return context.CurrentParseTree; }
private void Parse(Grammar grammar, string script, bool expectError) { var compiler = new Compiler(grammar); var context = new CompilerContext(compiler); var source = new SourceStream(script, "source"); AstNode root = compiler.Parse(context, source); compiler.AnalyzeCode(root, context); if (!expectError && context.Errors.Count > 0) Assert.Fail("Unexpected source error(s) found: " + context.Errors[0].Message); }
public EditorAdapter(Compiler compiler) { _compiler = compiler; _context = new CompilerContext(_compiler); _scanner = compiler.Parser.Scanner; _scanner.BeginScan(_context); _parseTree = new ParseTree(string.Empty, "Source"); _colorizerThread = new Thread(ColorizerLoop); _colorizerThread.IsBackground = true; _parserThread = new Thread(ParserLoop); _parserThread.IsBackground = true; }
public void BeginScan(CompilerContext context) { _context = context; _bufferedTokens.Clear(); //create streams FilteredStream = UnfilteredStream = CreateUnfilteredTokenStream(); //chain all token filters foreach (TokenFilter filter in _data.TokenFilters) { FilteredStream = filter.BeginFiltering(context, FilteredStream); } FilteredTokenEnumerator = FilteredStream.GetEnumerator(); }
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) { //TODO: fix this. // This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal. // if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter, // not by terminal. context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v"; _prevLine = 0; _indents.Clear(); foreach (Token token in tokens) { if (token.Terminal == base.Grammar.Eof) { yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location); //this is necessary, because grammar rules reference newLine terminator //unindent all buffered indents if (_trackIndents) foreach (int i in _indents) yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location); _indents.Clear(); //return EOF token yield return token; yield break; }//if Eof //Now deal with normal, non-EOF tokens //We intercept only content tokens on new lines if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) { yield return token; continue; } //if we are here, we have content token on new line; produce newLine token and possibly indents yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location); _prevLine = token.Location.Line; if (!_trackIndents) { yield return token; continue; } //Now take care of indents int currIndent = token.Location.Column; int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek(); if (currIndent > prevIndent) { _indents.Push(currIndent); yield return CreateSpecialToken(GrammarData.Grammar.Indent, token.Location); } else if (currIndent < prevIndent) { //produce one or more dedent tokens while popping indents from stack while (_indents.Count > 0 && _indents.Peek() > currIndent) { _indents.Pop(); yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location); } if (_indents.Count == 0 || _indents.Peek() != currIndent) { yield return context.CreateErrorTokenAndReportError (token.Location, string.Empty, "Invalid dedent level, no previous matching indent found."); //TODO: add error recovery here } }//else if currIndent < prevIndent yield return token; } //foreach token }//method
public override Token TryMatch(CompilerContext context, ISourceStream source) { char current = source.CurrentChar; if (!LineTerminators.Contains(current)) return null; //Treat \r\n as a single terminator bool doExtraShift = (current == '\r' && source.NextChar == '\n'); source.Position++; //main shift if (doExtraShift) source.Position++; Token result = new Token(this, source.TokenStart, source.GetLexeme(), null); return result; }
public void Parse(CompilerContext context) { _context = context; _traceOn = _context.OptionIsSet(CompilerOptions.TraceParser); _currentInput = null; InputStack.Clear(); Stack.Clear(); _currentState = Data.InitialState; //set the current state to InitialState Stack.Push(new ParseTreeNode(Data.InitialState)); //main loop while (ExecuteAction()) {} }//Parse
private void Evaluate(Grammar grammar, string script, bool expectError, object result) { var compiler = new Compiler(grammar); var context = new CompilerContext(compiler); var source = new SourceStream(script, "source"); AstNode root = compiler.Parse(context, source); compiler.AnalyzeCode(root, context); if (!expectError && context.Errors.Count > 0) Assert.Fail("Unexpected source error(s) found: " + context.Errors[0].Message); var evalContext = new EvaluationContext(context.Runtime, root); root.Evaluate(evalContext); Assert.AreEqual(evalContext.CurrentResult, result, "Evaluation result is null, expected " + result); }
public virtual string GetSyntaxErrorMessage(CompilerContext context, ParserState state, ParseTreeNode currentInput) { return null; //Irony then would construct default message }
public virtual void CreateAstNode(CompilerContext context, ParseTreeNode nodeInfo) { var term = nodeInfo.Term; if (term.NodeCreator != null) { term.NodeCreator(context, nodeInfo); //We assume that Node creator method creates node and initializes it, so parser does not need to call // IAstNodeInit.InitNode() method on node object. return; } Type nodeType = term.NodeType ?? this.DefaultNodeType; if (nodeType == null) return; nodeInfo.AstNode = Activator.CreateInstance(nodeType); //Initialize node var iInit = nodeInfo.AstNode as IAstNodeInit; if (iInit != null) iInit.Init(context, nodeInfo); }
//This method is called if Scanner failed to produce token; it offers custom method a chance public virtual Token TryMatch(CompilerContext context, ISourceStream source) { return null; }
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens) { //TODO: fix this. // This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal. // if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter, // not by terminal. context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v"; _prevLine = 0; _indents.Clear(); foreach (Token token in tokens) { if (token.Terminal == base.Grammar.Eof) { yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location)); //this is necessary, because grammar rules reference newLine terminator //unindent all buffered indents if (_trackIndents) { foreach (int i in _indents) { yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location)); } } _indents.Clear(); //return EOF token yield return(token); yield break; }//if Eof //Now deal with normal, non-EOF tokens //We intercept only content tokens on new lines if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) { yield return(token); continue; } //if we are here, we have content token on new line; produce newLine token and possibly indents yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location)); _prevLine = token.Location.Line; if (!_trackIndents) { yield return(token); continue; } //Now take care of indents int currIndent = token.Location.Column; int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek(); if (currIndent > prevIndent) { _indents.Push(currIndent); yield return(CreateSpecialToken(GrammarData.Grammar.Indent, token.Location)); } else if (currIndent < prevIndent) { //produce one or more dedent tokens while popping indents from stack while (_indents.Count > 0 && _indents.Peek() > currIndent) { _indents.Pop(); yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location)); } if (_indents.Count == 0 || _indents.Peek() != currIndent) { yield return(context.CreateErrorTokenAndReportError(token.Location, string.Empty, "Invalid dedent level, no previous matching indent found.")); //TODO: add error recovery here } } //else if currIndent < prevIndent yield return(token); } //foreach token } //method
protected override void InitDetails(CompilerContext context, CompoundTokenDetails details) { base.InitDetails(context, details); details.Flags = (short) this.Flags; }
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9 // so we try to do a quick parse for these, without starting the whole general process protected override Token QuickParse(CompilerContext context, ISourceStream source) { if (IsSet(NumberFlags.DisableQuickParse)) return null; char current = source.CurrentChar; if (char.IsDigit(current) && QuickParseTerminators.IndexOf(source.NextChar) >= 0) { int iValue = current - '0'; object value = null; switch (DefaultIntTypes[0]) { case TypeCode.Int32: value = iValue; break; case TypeCode.UInt32: value = (UInt32)iValue; break; case TypeCode.Byte: value = (byte)iValue; break; case TypeCode.SByte: value = (sbyte) iValue; break; case TypeCode.Int16: value = (Int16)iValue; break; case TypeCode.UInt16: value = (UInt16)iValue; break; default: return null; } Token token = new Token(this, source.TokenStart, current.ToString(), value); source.Position++; return token; } else return null; }
public virtual IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens) { yield break; }
public override Token TryMatch(CompilerContext context, ISourceStream source) { return _handler(this, context, source); }
protected override void InitDetails(CompilerContext context, CompoundTokenDetails details) { base.InitDetails(context, details); details.Flags = (short)this.Flags; }
protected virtual Token QuickParse(CompilerContext context, ISourceStream source) { return(null); }
public override Token TryMatch(CompilerContext context, ISourceStream source) { return(_handler(this, context, source)); }
//This method is called if Scanner failed to produce token; it offers custom method a chance public virtual Token TryMatch(CompilerContext context, ISourceStream source) { return(null); }
protected virtual void InitDetails(CompilerContext context, CompoundTokenDetails details) { details.PartialOk = (context.Mode == CompileMode.VsLineScan); details.PartialContinues = (context.ScannerState.Value != 0); }
public virtual string GetSyntaxErrorMessage(CompilerContext context, ParserState state, ParseTreeNode currentInput) { return(null); //Irony then would construct default message }