|
public static CreateSyntaxErrorToken ( Irony.Compiler.CompilerContext context, |
||
| context | Irony.Compiler.CompilerContext | |
| location | ||
| message | string | |
| return | Token | |
//TODO: need to rewrite, looks ugly
private bool Recover()
{
if (_currentToken.Category != TokenCategory.Error)
{
_currentToken = Grammar.CreateSyntaxErrorToken(_context, _currentToken.Location, "Syntax error.");
}
//Check the current state and states in stack for error shift action - this would be recovery state.
ActionRecord action = GetCurrentAction();
if (action == null || action.ActionType == ParserActionType.Reduce)
{
while (Stack.Count > 0)
{
_currentState = Stack.Top.State;
Stack.Pop(1);
action = GetCurrentAction();
if (action != null && action.ActionType != ParserActionType.Reduce)
{
break; //we found shift action for error token
}
}//while
}//if
if (action == null || action.ActionType == ParserActionType.Reduce)
{
return(false); //could not find shift action, cannot recover
}
//We found recovery state, and action contains ActionRecord for "error shift". Lets shift it.
ExecuteShiftAction(action);//push the error token
// Now shift all tokens from input that can be shifted.
// These are the ones that are found in error production after the error. We ignore all other tokens
// We stop when we find a state with reduce-only action.
while (_currentToken.Terminal != Grammar.Eof)
{
//with current token, see if we can shift it.
action = GetCurrentAction();
if (action == null)
{
NextToken(); //skip this token and continue reading input
continue;
}
if (action.ActionType == ParserActionType.Reduce || action.ActionType == ParserActionType.Operator)
{
//we can reduce - let's reduce and return success - we recovered.
ExecuteReduceAction(action);
return(true);
}
//it is shift action, let's shift
ExecuteShiftAction(action);
}//while
return(false); //
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
bool ignoreCase = !Grammar.CaseSensitive;
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, ignoreCase))
{
return(null);
}
//Find end symbol
source.Position += StartSymbol.Length;
while (!source.EOF())
{
int firstCharPos;
if (EndSymbols.Count == 1)
{
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
}
else
{
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
}
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
{
return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
}
else
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
}
}
//We found a character that might start an end symbol; let's see if it is true.
source.Position = firstCharPos;
foreach (string endSymbol in EndSymbols)
{
if (source.MatchSymbol(endSymbol, ignoreCase))
{
//We found end symbol
source.Position += endSymbol.Length;
return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
}//if
}
source.Position++; //move to the next char and try again
} //while
return(null); //never happens
} //method
public bool BuildPairsList = false; //do not build pairs list
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
foreach (Token token in tokens)
{
if (!token.Term.IsSet(TermOptions.IsBrace))
{
yield return(token);
continue;
}
//open brace symbol
if (token.Term.IsSet(TermOptions.IsOpenBrace))
{
_braces.Push(token);
yield return(token);
continue;
}
//We have closing brace
if (_braces.Count == 0)
{
yield return(Grammar.CreateSyntaxErrorToken(context, token.Span.Start,
"Unmatched closing brace '{0}'", token.Text));
continue;
}
//check match
Token last = _braces.Pop();
if (last.Symbol.IsPairFor != token.Symbol)
{
yield return(Grammar.CreateSyntaxErrorToken(context, token.Span.Start,
"Unmatched closing brace '{0}' - expected '{1}'", token.Text, last.Symbol.IsPairFor.Name));
continue;
}
//everything is ok, there's matching brace on top of the stack
if (BuildPairsList)
{
BracePairs.Add(new BracePair(last, token));
}
yield return(token); //return this token
}//foreach token
yield break;
} //method
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token token = null;
if (IsSet(TermOptions.EnableQuickParse))
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
}
source.Position = source.TokenStart.Position;
ScanDetails details = new ScanDetails();
details.Flags = DefaultFlags;
details.TypeCodes = _defaultTypes;
ReadPrefix(source, details);
if (!ReadBody(source, details))
{
return(null);
}
if (details.HasError())
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, details.Error));
}
ReadSuffix(source, details);
if (!ConvertValue(details))
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value: " + details.Error));
}
token = CreateToken(context, source, details);
return(token);
}
private Token ReadToken()
{
if (_bufferedTokens.Count > 0)
{
Token tkn = _bufferedTokens[0];
_bufferedTokens.RemoveAt(0);
return(tkn);
}
//1. Skip whitespace. We don't need to check for EOF: at EOF we start getting 0-char, so we'll get out automatically
while (_data.Grammar.WhitespaceChars.IndexOf(_source.CurrentChar) >= 0)
{
_source.Position++;
}
//That's the token start, calc location (line and column)
SetTokenStartLocation();
//Check for EOF
if (_source.EOF())
{
return(Token.Create(Grammar.Eof, _context, _source.TokenStart, string.Empty, Grammar.Eof.Name));
}
//Find matching terminal
// First, try terminals with explicit "first-char" prefixes, selected by current char in source
TerminalList terms = SelectTerminals(_source.CurrentChar);
Token result = MatchTerminals(terms);
//If no token, try FallbackTerminals
if (result == null && _data.FallbackTerminals.Count > 0)
{
result = MatchTerminals(_data.FallbackTerminals);
}
//If we don't have a token from registered terminals, try Grammar's method
if (result == null)
{
result = _data.Grammar.TryMatch(_context, _source);
}
//Check if we have a multi-token; if yes, copy all but first child tokens from ChildNodes to _bufferedTokens,
// and set result to the first child token
if (result != null && result.IsMultiToken())
{
foreach (Token tkn in result.ChildNodes)
{
_bufferedTokens.Add(tkn);
}
result = _bufferedTokens[0];
_bufferedTokens.RemoveAt(0);
}
//If we have normal token then return it
if (result != null && !result.IsError())
{
//restore position to point after the result token
_source.Position = _source.TokenStart.Position + result.Length;
return(result);
}
//we have an error: either error token or no token at all
if (result == null) //if no error result then create it
{
result = Grammar.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);
}
Recover();
return(result);
}//method
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens)
{
if (token.Terminal == Grammar.Eof)
{
yield return(CreateSpecialToken(Grammar.NewLine, context, token.Location)); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
{
foreach (int i in _indents)
{
yield return(CreateSpecialToken(Grammar.Dedent, context, token.Location));
}
}
_indents.Clear();
//return EOF token
yield return(token);
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine)
{
yield return(token);
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return(CreateSpecialToken(Grammar.NewLine, context, token.Location));
_prevLine = token.Location.Line;
if (!_trackIndents)
{
yield return(token);
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent)
{
_indents.Push(currIndent);
yield return(CreateSpecialToken(Grammar.Indent, context, token.Location));
}
else if (currIndent < prevIndent)
{
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent)
{
_indents.Pop();
yield return(CreateSpecialToken(Grammar.Dedent, context, token.Location));
}
if (_indents.Count == 0 || _indents.Peek() != currIndent)
{
yield return(Grammar.CreateSyntaxErrorToken(context, token.Location,
"Invalid dedent level, no previous matching indent found."));
//TODO: add error recovery here
}
} //else if currIndent < prevIndent
yield return(token);
} //foreach token
} //method
