public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens)
{
foreach (Token token in tokens) {
if (!token.Term.IsSet(TermOptions.IsBrace)) {
yield return token;
continue;
}
//open brace symbol
if (token.Term.IsSet(TermOptions.IsOpenBrace)) {
_braces.Push(token);
yield return token;
continue;
}
//We have closing brace
if (_braces.Count == 0) {
yield return context.CreateErrorTokenAndReportError( token.Span.Start, token.Text, "Unmatched closing brace '{0}'", token.Text);
continue;
}
//check match
Token last = _braces.Pop();
if (last.Symbol.IsPairFor != token.Symbol) {
yield return context.CreateErrorTokenAndReportError(token.Span.Start, token.Text,
"Unmatched closing brace '{0}' - expected '{1}'", last.Symbol.IsPairFor.Name);
continue;
}
//everything is ok, there's matching brace on top of the stack
Token.LinkMatchingBraces(last, token);
yield return token; //return this token
}//foreach token
yield break;
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token result;
if (context.ScannerState.Value != 0) {
// we are continuing in line mode - restore internal env (none in this case)
context.ScannerState.Value = 0;
} else {
//we are starting from scratch
if (!BeginMatch(context, source)) return null;
}
result = CompleteMatch(context, source);
if (result != null) return result;
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
return Token.Create(this, context, source.TokenStart, source.GetLexeme());
if (context.Mode == CompileMode.VsLineScan)
return CreateIncompleteToken(context, source);
return context.CreateErrorTokenAndReportError(source.TokenStart, string.Empty, "Unclosed comment block");
}
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens)
{
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens) {
if (token.Terminal == Grammar.Eof) {
yield return CreateSpecialToken(Grammar.NewLine, context, token.Location); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
foreach (int i in _indents)
yield return CreateSpecialToken(Grammar.Dedent, context, token.Location);
_indents.Clear();
//return EOF token
yield return token;
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) {
yield return token;
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return CreateSpecialToken(Grammar.NewLine, context, token.Location);
_prevLine = token.Location.Line;
if (!_trackIndents) {
yield return token;
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent) {
_indents.Push(currIndent);
yield return CreateSpecialToken(Grammar.Indent, context, token.Location);
} else if (currIndent < prevIndent) {
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent) {
_indents.Pop();
yield return CreateSpecialToken(Grammar.Dedent, context, token.Location);
}
if (_indents.Count == 0 || _indents.Peek() != currIndent) {
yield return context.CreateErrorTokenAndReportError (token.Location, string.Empty, "Invalid dedent level, no previous matching indent found.");
//TODO: add error recovery here
}
}//else if currIndent < prevIndent
yield return token;
} //foreach token
}