public Token (Token t1, Token t2, TokenType type) { Owner = t1.Owner; Text = t1.Text + t2.Text; Offset = t1.Offset; Line = t1.Line; Column = t1.Column; Type = type; }
public UnexpectedTokenException (Token t, string expectedToken) { this.Token = t; this.ExpectedToken = expectedToken; }
public void PutBack (bool reverse, params Token [] ts) { Token [] rev_ts = ts; if (reverse) { rev_ts = new Token [ts.Length]; for (int x = 0; x < ts.Length; ++x) rev_ts [x] = ts [ts.Length - x - 1]; } foreach (Token t in rev_ts) _Queue.Add (t); }
private Token _Read () { if (Text.Length == 0) return null; if (StringMode) { return _StringRead (); } int startCaret = _Caret; int startLine = _Line; int startCol = _Column; if (_Caret >= Text.Length) return null; if (_Caret < 0) _Caret = 0; char c = Text [_Caret]; Token t = new Token (this, c, _Caret, _Line, _Column); if (StringMode) return t; // it has just switched to, so return this and await string parsing. if (c == '\n') { ++_Line; _Column = 0; } ++_Column; ++_Caret; if (_Caret >= Text.Length) { return t; } c = Text [_Caret]; // try to combine the tokens while (_Caret < Text.Length) { Token newt = t + new Token (this, c, _Caret, _Line, _Column); if (c == '\n') { ++_Line; _Column = 0; } if (newt == null) { if (t.Type == TokenType.MaybeSpecial) { // We may have parsed into multiple tokens // in our search for what we thought was a // special token. Now we must disable special // tokens and reparse from where we started bool old = _IgnoreSpecial; _IgnoreSpecial = true; _Caret = startCaret; _Line = startLine; _Column = startCol; try { return Read (); } finally { _IgnoreSpecial = old; } } return t; } t = newt; ++_Column; ++_Caret; if (_Caret < Text.Length) c = Text [_Caret]; } return t; }
public void PutBack (Token t) { _Queue.Add (t); }