public override void Reset()
{
Indents.Clear();
Indents.Push(0);
OutputTokens.Clear();
PreviousToken = null;
CurrentToken = null;
PreviousTokenLocation = new SourceLocation();
}
public IToken <INode> PeekToken()
{
if (Store.IsNull())
{
READ_LINE_:
var line = BaseReader.LineNumber;
var ts = ReadLineTokens(BaseReader, Parser?.TokenStack.LastOrNull());
if (ts.First().Type == Symbols.EOL)
{
goto READ_LINE_;
}
Store.AddRange(ts);
if (!Store[0].EndOfToken && (Indents.Count == 0 || Store[0].Indent > Indents.Peek()))
{
Store.Insert(0, new Token {
Type = Symbols.BEGIN, LineNumber = line, Indent = Store[0].Indent
});
Indents.Push(Store[0].Indent);
}
else
{
var count = 0;
var head = Store[0];
while (Indents.Count > 0 && (head.EndOfToken || head.Indent < Indents.Peek()))
{
Store.Insert(count, new Token {
Type = Symbols.END, LineNumber = line, Indent = Indents.Pop()
});
count++;
}
}
}
READ_FIRST_:
var first = Store.First();
if (Parser is { } && first.Type == Symbols.EOL && !Parser.IsAccept(first))
public void ProcessToken(Token token)
{
SetCurrentToken(token);
//Quick checks
if (_isContinuation)
{
return;
}
var tokenTerm = token.Terminal;
//check EOF
if (tokenTerm == _grammar.Eof)
{
ProcessEofToken();
return;
}
if (tokenTerm != _grammar.LineStartTerminal)
{
return;
}
//if we are here, we have LineStart token on new line; first remove it from stream, it should not go to parser
OutputTokens.Pop();
if (PreviousToken == null)
{
return;
}
// first check if there was continuation symbol before
// or - if checkBraces flag is set - check if there were open braces
if (_prevIsContinuation || _checkBraces && _context.OpenBraces.Count > 0)
{
return; //no Eos token in this case
}
if (_prevIsOperator && _checkOperator)
{
return; //no Eos token in this case
}
//We need to produce Eos token and indents (if _produceIndents is set).
// First check indents - they go first into OutputTokens stack, so they will be popped out last
if (_produceIndents)
{
var currIndent = token.Location.Column;
var prevIndent = Indents.Peek();
if (currIndent > prevIndent)
{
Indents.Push(currIndent);
PushOutlineToken(_grammar.Indent, token.Location);
}
else if (currIndent < prevIndent)
{
PushDedents(currIndent);
//check that current indent exactly matches the previous indent
if (Indents.Peek() != currIndent)
{
//fire error
OutputTokens.Push(new Token(_grammar.SyntaxError, token.Location, string.Empty, Resources.ErrInvDedent));
// "Invalid dedent level, no previous matching indent found."
}
}
}
//Finally produce Eos token, but not in command line mode. In command line mode the Eos was already produced
// when we encountered Eof on previous line
if (_context.Mode != ParseMode.CommandLine)
{
var eosLocation = ComputeEosLocation();
PushOutlineToken(_grammar.Eos, eosLocation);
}
}
