public override Token?NextToken()
{
_current = LLNextToken();
if (_current == null)
{
return(null);
}
TK tt = _current.Value.Kind;
if (IsOpener(tt))
{
var v = _current.Value;
GatherChildren(ref v);
return(_current = v);
}
else
{
return(_current);
}
}
public override Maybe <Token> NextToken()
{
_current = LLNextToken();
if (!_current.HasValue)
{
return(_current);
}
TK tt = _current.Value.Kind;
if (Token.IsOpener(tt))
{
var v = _current.Value;
GatherChildren(ref v);
return(_current = v);
}
else
{
return(_current);
}
}
void GatherChildren(ref Token openToken)
{
Debug.Assert(openToken.Value == null);
if (openToken.Value != null && openToken.Children != null)
{
return; // wtf, it's already a tree
}
TK ott = openToken.Kind;
int oldIndentLevel = _source.IndentLevel;
TokenTree children = new TokenTree(_source.SourceFile);
for (;;)
{
Token?t = LLNextToken(); // handles LBrace, LParen, LBrack internally
if (t == null)
{
WriteError(openToken.StartIndex, "Reached end-of-file before '{0}' was closed", openToken);
break;
}
TK tt = t.Value.Kind;
if (IsOpener(tt))
{
var v = t.Value;
GatherChildren(ref v);
children.Add(v);
if (_closer != null && _closerMatched)
{
children.Add(_closer.Value);
_closer = null;
}
}
else if (IsCloser(tt))
{
// indent must match dedent, '{' must match '}' (the parser
// can complain itself about "(]" and "[)" if it wants; we
// allow these to match because some languages might want it.)
bool dentMismatch = (ott == TK.Indent) != (tt == TK.Dedent);
if (dentMismatch || (ott == TK.LBrace) != (tt == TK.RBrace))
{
WriteError(openToken.StartIndex, "Opening '{0}' does not match closing '{1}' on line {2}",
openToken.ToString(), t.Value.ToString(), SourceFile.IndexToLine(t.Value.StartIndex).Line);
// - If dentMismatch and ott == TK.Indent, do not close.
// - If dentMismatch and tt = TK.Dedent, close but do not match.
// - If the closer is more indented than the opener, do not close.
// - If the closer is less indented than the opener, close but do not match.
// - If the closer is the same indentation as the opener, close and match.
if (dentMismatch ? tt == TK.Dedent : IndentLevel <= oldIndentLevel)
{
// close
_closer = t.Value;
_closerMatched = !dentMismatch && (IndentLevel == oldIndentLevel);
break;
}
else
{
children.Add(t.Value); // do not close
}
}
else
{
_closer = t.Value;
_closerMatched = true;
break;
}
}
else
{
children.Add(t.Value);
}
}
openToken.Value = children;
}
private bool IsCloser(TK tt)
{
return(tt >= TK.LParen && ((int)tt & 0x0100) != 0);
}
private bool IsOpener(TK tt)
{
return(tt >= TK.LParen && ((int)tt & 0x0100) == 0);
}
