public override TokenAst TryMatch(CompilerContext context, ISourceStream source) { if (!source.MatchSymbol(_startSymbol, false)) { return(null); } source.Position += _startSymbol.Length; while (!source.EOF()) { int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position); if (firstCharPos < 0) { source.Position = source.Text.Length; return(_isLineComment ? TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()) : LRParser.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block")); } source.Position = firstCharPos; if (source.MatchSymbol(_endSymbol, false)) { source.Position += _endSymbol.Length; return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())); } source.Position++; } throw new NotSupportedException(); }
private TokenAst ReadToken() { if (_bufferedTokens.Count > 0) { TokenAst tkn = _bufferedTokens[0]; _bufferedTokens.RemoveAt(0); return(tkn); } SkipWhiteSpaces(); SetTokenStartLocation(); if (_source.EOF()) { return(TokenAst.Create(LRParser.Eof, _context, _source.TokenStart, string.Empty, LRParser.Eof.Name)); } IEnumerable <ITerminal> terms = SelectTerminals(_source.CurrentChar); TokenAst result = MatchTerminals(terms); if (result != null && !result.IsError()) { _source.Position = _source.TokenStart.Position + result.Length; return(result); } if (result != null) { return(result); } result = LRParser.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar); return(result); }