private TokenAst ReadToken() { if (_bufferedTokens.Count > 0) { TokenAst tkn = _bufferedTokens[0]; _bufferedTokens.RemoveAt(0); return(tkn); } SkipWhiteSpaces(); SetTokenStartLocation(); if (_source.EOF()) { return(TokenAst.Create(Grammar.Eof, _context, _source.TokenStart, string.Empty, Grammar.Eof.Name)); } TerminalList terms = SelectTerminals(_source.CurrentChar); TokenAst result = MatchTerminals(terms); if (result != null && !result.IsError()) { _source.Position = _source.TokenStart.Position + result.Length; return(result); } if (result == null) { result = Grammar.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar); } return(result); }
public override TokenAst TryMatch(CompilerContext context, ISourceStream source) { if (!source.MatchSymbol(_startSymbol, false)) { return(null); } source.Position += _startSymbol.Length; while (!source.EOF()) { int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position); if (firstCharPos < 0) { source.Position = source.Text.Length; if (_isLineComment) { return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())); } else { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block")); } } source.Position = firstCharPos; if (source.MatchSymbol(_endSymbol, false)) { source.Position += _endSymbol.Length; return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())); } source.Position++; } throw new NotSupportedException(); }