public static CreateSyntaxErrorToken ( Irony.Compiler.CompilerContext context, |
||
context | Irony.Compiler.CompilerContext | |
location | ||
message | string | |
리턴 | Token |
//TODO: need to rewrite, looks ugly private bool Recover() { if (_currentToken.Category != TokenCategory.Error) { _currentToken = Grammar.CreateSyntaxErrorToken(_context, _currentToken.Location, "Syntax error."); } //Check the current state and states in stack for error shift action - this would be recovery state. ActionRecord action = GetCurrentAction(); if (action == null || action.ActionType == ParserActionType.Reduce) { while (Stack.Count > 0) { _currentState = Stack.Top.State; Stack.Pop(1); action = GetCurrentAction(); if (action != null && action.ActionType != ParserActionType.Reduce) { break; //we found shift action for error token } }//while }//if if (action == null || action.ActionType == ParserActionType.Reduce) { return(false); //could not find shift action, cannot recover } //We found recovery state, and action contains ActionRecord for "error shift". Lets shift it. ExecuteShiftAction(action);//push the error token // Now shift all tokens from input that can be shifted. // These are the ones that are found in error production after the error. We ignore all other tokens // We stop when we find a state with reduce-only action. while (_currentToken.Terminal != Grammar.Eof) { //with current token, see if we can shift it. action = GetCurrentAction(); if (action == null) { NextToken(); //skip this token and continue reading input continue; } if (action.ActionType == ParserActionType.Reduce || action.ActionType == ParserActionType.Operator) { //we can reduce - let's reduce and return success - we recovered. ExecuteReduceAction(action); return(true); } //it is shift action, let's shift ExecuteShiftAction(action); }//while return(false); // }
public override Token TryMatch(CompilerContext context, ISourceStream source) { bool ignoreCase = !Grammar.CaseSensitive; //Check starting symbol if (!source.MatchSymbol(StartSymbol, ignoreCase)) { return(null); } //Find end symbol source.Position += StartSymbol.Length; while (!source.EOF()) { int firstCharPos; if (EndSymbols.Count == 1) { firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position); } else { firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position); } if (firstCharPos < 0) { source.Position = source.Text.Length; if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end. { return(Token.Create(this, context, source.TokenStart, source.GetLexeme())); } else { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block")); } } //We found a character that might start an end symbol; let's see if it is true. source.Position = firstCharPos; foreach (string endSymbol in EndSymbols) { if (source.MatchSymbol(endSymbol, ignoreCase)) { //We found end symbol source.Position += endSymbol.Length; return(Token.Create(this, context, source.TokenStart, source.GetLexeme())); }//if } source.Position++; //move to the next char and try again } //while return(null); //never happens } //method
public bool BuildPairsList = false; //do not build pairs list public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens) { foreach (Token token in tokens) { if (!token.Term.IsSet(TermOptions.IsBrace)) { yield return(token); continue; } //open brace symbol if (token.Term.IsSet(TermOptions.IsOpenBrace)) { _braces.Push(token); yield return(token); continue; } //We have closing brace if (_braces.Count == 0) { yield return(Grammar.CreateSyntaxErrorToken(context, token.Span.Start, "Unmatched closing brace '{0}'", token.Text)); continue; } //check match Token last = _braces.Pop(); if (last.Symbol.IsPairFor != token.Symbol) { yield return(Grammar.CreateSyntaxErrorToken(context, token.Span.Start, "Unmatched closing brace '{0}' - expected '{1}'", token.Text, last.Symbol.IsPairFor.Name)); continue; } //everything is ok, there's matching brace on top of the stack if (BuildPairsList) { BracePairs.Add(new BracePair(last, token)); } yield return(token); //return this token }//foreach token yield break; } //method
public override Token TryMatch(CompilerContext context, ISourceStream source) { Token token = null; if (IsSet(TermOptions.EnableQuickParse)) { token = QuickParse(context, source); if (token != null) { return(token); } } source.Position = source.TokenStart.Position; ScanDetails details = new ScanDetails(); details.Flags = DefaultFlags; details.TypeCodes = _defaultTypes; ReadPrefix(source, details); if (!ReadBody(source, details)) { return(null); } if (details.HasError()) { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, details.Error)); } ReadSuffix(source, details); if (!ConvertValue(details)) { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value: " + details.Error)); } token = CreateToken(context, source, details); return(token); }
private Token ReadToken() { if (_bufferedTokens.Count > 0) { Token tkn = _bufferedTokens[0]; _bufferedTokens.RemoveAt(0); return(tkn); } //1. Skip whitespace. We don't need to check for EOF: at EOF we start getting 0-char, so we'll get out automatically while (_data.Grammar.WhitespaceChars.IndexOf(_source.CurrentChar) >= 0) { _source.Position++; } //That's the token start, calc location (line and column) SetTokenStartLocation(); //Check for EOF if (_source.EOF()) { return(Token.Create(Grammar.Eof, _context, _source.TokenStart, string.Empty, Grammar.Eof.Name)); } //Find matching terminal // First, try terminals with explicit "first-char" prefixes, selected by current char in source TerminalList terms = SelectTerminals(_source.CurrentChar); Token result = MatchTerminals(terms); //If no token, try FallbackTerminals if (result == null && _data.FallbackTerminals.Count > 0) { result = MatchTerminals(_data.FallbackTerminals); } //If we don't have a token from registered terminals, try Grammar's method if (result == null) { result = _data.Grammar.TryMatch(_context, _source); } //Check if we have a multi-token; if yes, copy all but first child tokens from ChildNodes to _bufferedTokens, // and set result to the first child token if (result != null && result.IsMultiToken()) { foreach (Token tkn in result.ChildNodes) { _bufferedTokens.Add(tkn); } result = _bufferedTokens[0]; _bufferedTokens.RemoveAt(0); } //If we have normal token then return it if (result != null && !result.IsError()) { //restore position to point after the result token _source.Position = _source.TokenStart.Position + result.Length; return(result); } //we have an error: either error token or no token at all if (result == null) //if no error result then create it { result = Grammar.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar); } Recover(); return(result); }//method
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens) { _prevLine = 0; _indents.Clear(); foreach (Token token in tokens) { if (token.Terminal == Grammar.Eof) { yield return(CreateSpecialToken(Grammar.NewLine, context, token.Location)); //this is necessary, because grammar rules reference newLine terminator //unindent all buffered indents if (_trackIndents) { foreach (int i in _indents) { yield return(CreateSpecialToken(Grammar.Dedent, context, token.Location)); } } _indents.Clear(); //return EOF token yield return(token); yield break; }//if Eof //Now deal with normal, non-EOF tokens //We intercept only content tokens on new lines if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) { yield return(token); continue; } //if we are here, we have content token on new line; produce newLine token and possibly indents yield return(CreateSpecialToken(Grammar.NewLine, context, token.Location)); _prevLine = token.Location.Line; if (!_trackIndents) { yield return(token); continue; } //Now take care of indents int currIndent = token.Location.Column; int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek(); if (currIndent > prevIndent) { _indents.Push(currIndent); yield return(CreateSpecialToken(Grammar.Indent, context, token.Location)); } else if (currIndent < prevIndent) { //produce one or more dedent tokens while popping indents from stack while (_indents.Count > 0 && _indents.Peek() > currIndent) { _indents.Pop(); yield return(CreateSpecialToken(Grammar.Dedent, context, token.Location)); } if (_indents.Count == 0 || _indents.Peek() != currIndent) { yield return(Grammar.CreateSyntaxErrorToken(context, token.Location, "Invalid dedent level, no previous matching indent found.")); //TODO: add error recovery here } } //else if currIndent < prevIndent yield return(token); } //foreach token } //method