public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) {
foreach (Token token in tokens) {
if (!token.Terminal.IsSet(TermOptions.IsBrace)) {
yield return token;
continue;
}
//open brace symbol
if (token.Terminal.IsSet(TermOptions.IsOpenBrace)) {
_braces.Push(token);
yield return token;
continue;
}
//We have closing brace
if (_braces.Count == 0) {
yield return context.CreateErrorTokenAndReportError( token.Location, token.Text, "Unmatched closing brace '{0}'", token.Text);
continue;
}
//check match
Token last = _braces.Pop();
if (last.AsSymbol.IsPairFor != token.AsSymbol) {
yield return context.CreateErrorTokenAndReportError(token.Location, token.Text,
"Unmatched closing brace '{0}' - expected '{1}'", last.AsSymbol.IsPairFor.Name);
continue;
}
//everything is ok, there's matching brace on top of the stack
Token.LinkMatchingBraces(last, token);
context.CurrentParseTree.OpenBraces.Add(last);
yield return token; //return this token
}//foreach token
yield break;
}//method
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token token;
//Try quick parse first, but only if we're not continuing
if (context.ScannerState.Value == 0)
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
source.Position = source.TokenStart.Position; //revert the position
}
CompoundTokenDetails details = new CompoundTokenDetails();
InitDetails(context, details);
if (context.ScannerState.Value == 0)
{
ReadPrefix(source, details);
}
if (!ReadBody(source, details))
{
return(null);
}
if (details.Error != null)
{
return(context.CreateErrorTokenAndReportError(source.TokenStart, source.CurrentChar.ToString(), details.Error));
}
if (details.IsPartial)
{
details.Value = details.Body;
}
else
{
ReadSuffix(source, details);
if (!ConvertValue(details))
{
return(context.CreateErrorTokenAndReportError(source.TokenStart, source.CurrentChar.ToString(), "Failed to convert the value: " + details.Error));
}
}
token = CreateToken(context, source, details);
if (details.IsPartial)
{
//Save terminal state so we can continue
context.ScannerState.TokenSubType = (byte)details.SubTypeIndex;
context.ScannerState.TerminalFlags = (short)details.Flags;
context.ScannerState.TerminalIndex = this.MultilineIndex;
}
else
{
context.ScannerState.Value = 0;
}
return(token);
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token result;
if (context.ScannerState.Value != 0)
{
// we are continuing in line mode - restore internal env (none in this case)
context.ScannerState.Value = 0;
}
else
{
//we are starting from scratch
if (!BeginMatch(context, source))
{
return(null);
}
}
result = CompleteMatch(context, source);
if (result != null)
{
return(result);
}
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
{
return(new Token(this, source.TokenStart, source.GetLexeme(), null));
}
if (context.Mode == CompileMode.VsLineScan)
{
return(CreateIncompleteToken(context, source));
}
return(context.CreateErrorTokenAndReportError(source.TokenStart, string.Empty, "Unclosed comment block"));
}
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
foreach (Token token in tokens)
{
if (!token.Terminal.IsSet(TermOptions.IsBrace))
{
yield return(token);
continue;
}
//open brace symbol
if (token.Terminal.IsSet(TermOptions.IsOpenBrace))
{
_braces.Push(token);
yield return(token);
continue;
}
//We have closing brace
if (_braces.Count == 0)
{
yield return(context.CreateErrorTokenAndReportError(token.Location, token.Text, "Unmatched closing brace '{0}'", token.Text));
continue;
}
//check match
Token last = _braces.Pop();
if (last.AsSymbol.IsPairFor != token.AsSymbol)
{
yield return(context.CreateErrorTokenAndReportError(token.Location, token.Text,
"Unmatched closing brace '{0}' - expected '{1}'", last.AsSymbol.IsPairFor.Name));
continue;
}
//everything is ok, there's matching brace on top of the stack
Token.LinkMatchingBraces(last, token);
context.CurrentParseTree.OpenBraces.Add(last);
yield return(token); //return this token
}//foreach token
yield break;
}//method
private Token ReadToken()
{
if (_bufferedTokens.Count > 0)
{
return(ReadBufferedToken());
}
//1. Skip whitespace. We don't need to check for EOF: at EOF we start getting 0-char, so we'll get out automatically
while (_grammar.WhitespaceChars.IndexOf(_source.CurrentChar) >= 0)
{
_source.Position++;
}
//That's the token start, calc location (line and column)
ComputeNewTokenLocation();
//Check for EOF
if (_source.EOF())
{
return(CreateFinalToken());
}
//Find matching terminal
// First, try terminals with explicit "first-char" prefixes, selected by current char in source
var terms = SelectTerminals(_source.CurrentChar);
var token = MatchTerminals(terms);
//If no token, try FallbackTerminals
if (token == null && terms != _data.FallbackTerminals && _data.FallbackTerminals.Count > 0)
{
token = MatchTerminals(_data.FallbackTerminals);
}
//If we don't have a token from registered terminals, try Grammar's method
if (token == null)
{
token = _grammar.TryMatch(_context, _source);
}
if (token is MultiToken)
{
token = UnpackMultiToken(token);
}
//If we have normal token then return it
if (token != null && !token.IsError())
{
//set position to point after the result token
_source.Position = _source.TokenStart.Position + token.Length;
return(token);
}
//we have an error: either error token or no token at all
if (token == null) //if no token then create error token
{
token = _context.CreateErrorTokenAndReportError(_source.TokenStart, _source.CurrentChar.ToString(), "Invalid character: '{0}'", _source.CurrentChar);
}
Recover();
return(token);
}//method
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) {
//TODO: fix this.
// This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal.
// if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter,
// not by terminal.
context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v";
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens) {
if (token.Terminal == base.Grammar.Eof) {
yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
foreach (int i in _indents)
yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location);
_indents.Clear();
//return EOF token
yield return token;
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) {
yield return token;
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location);
_prevLine = token.Location.Line;
if (!_trackIndents) {
yield return token;
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent) {
_indents.Push(currIndent);
yield return CreateSpecialToken(GrammarData.Grammar.Indent, token.Location);
} else if (currIndent < prevIndent) {
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent) {
_indents.Pop();
yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location);
}
if (_indents.Count == 0 || _indents.Peek() != currIndent) {
yield return context.CreateErrorTokenAndReportError (token.Location, string.Empty, "Invalid dedent level, no previous matching indent found.");
//TODO: add error recovery here
}
}//else if currIndent < prevIndent
yield return token;
} //foreach token
}//method
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
//TODO: fix this.
// This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal.
// if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter,
// not by terminal.
context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v";
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens)
{
if (token.Terminal == base.Grammar.Eof)
{
yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location)); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
{
foreach (int i in _indents)
{
yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location));
}
}
_indents.Clear();
//return EOF token
yield return(token);
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine)
{
yield return(token);
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location));
_prevLine = token.Location.Line;
if (!_trackIndents)
{
yield return(token);
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent)
{
_indents.Push(currIndent);
yield return(CreateSpecialToken(GrammarData.Grammar.Indent, token.Location));
}
else if (currIndent < prevIndent)
{
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent)
{
_indents.Pop();
yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location));
}
if (_indents.Count == 0 || _indents.Peek() != currIndent)
{
yield return(context.CreateErrorTokenAndReportError(token.Location, string.Empty, "Invalid dedent level, no previous matching indent found."));
//TODO: add error recovery here
}
} //else if currIndent < prevIndent
yield return(token);
} //foreach token
} //method
