void CheckForEOF(antlr.IToken token) { if (antlr.Token.EOF_TYPE != token.Type) { return; } EnqueueEOS(token); while (CurrentIndentLevel > 0) { EnqueueDedent(); _indentStack.Pop(); } }
public void mDQUOTE(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = DQUOTE; match('\u0022'); if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
protected void mDIGIT(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = DIGIT; matchRange('\u0030', '\u0039'); if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mBINOP(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = BINOP; switch (cached_LA1) { case '|': { match('|'); break; } case '&': { match('&'); break; } case '^': { match('^'); break; } default: if ((cached_LA1 == '=') && (cached_LA2 == '>')) { match("=>"); } else if ((cached_LA1 == '=') && (true)) { match('='); } else { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } break; } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mWS(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = WS; { switch (cached_LA1) { case ' ': { match(' '); break; } case '\t': { match('\t'); break; } case '\n': { match('\n'); newline(); break; } case '\r': { match('\r'); break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } } _ttype = Token.SKIP; if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mCRLF(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = CRLF; mCR(false); mLF(false); newline(); if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
private Tuple <int, int> CalculateEndpoint(antlr.IToken token, int endLine, int endIndex, int delimiterLength) { var startIndex = positionMap[token.getLine() - 1][token.getColumn() - 1]; var startLine = token.getLine() - 1; if (startLine > endLine || startLine == endLine && startIndex > endIndex) { whitespaces.Add(new TextSpan { iStartLine = endLine, iStartIndex = endIndex, iEndLine = startLine, iEndIndex = startIndex }); } endLine = startLine - 1; endIndex = 0; var runningIndex = startIndex + delimiterLength; foreach (var part in token.getText().Split(new[] { "\r\n" }, StringSplitOptions.None)) { endLine++; endIndex = runningIndex + part.Length; runningIndex = 0; } endIndex += delimiterLength; //endIndex = positionMap[endLine][endIndex]; var cluster = new MappedToken( startLine * lineSize + startIndex, endIndex - startIndex); if (tokenMap.Count > 0 && tokenMap[tokenMap.Count() - 1].Index >= cluster.Index) { throw new ArgumentException("Token Mapping order"); } tokenMap.Add(cluster); return(new Tuple <int, int>(endLine, endIndex)); }
antlr.IToken BufferUntilNextNonWhiteSpaceToken() { antlr.IToken token = null; while (true) { token = _istream.nextToken(); int ttype = token.Type; if (antlr.Token.SKIP == ttype) { continue; } if (_wsTokenType == ttype) { _buffer.Append(token.getText()); continue; } break; } return(token); }
public void mLIT_CADENA(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = LIT_CADENA; int _saveIndex = 0; _saveIndex = text.Length; match('\''); text.Length = _saveIndex; { // ( ... )* for (;;) { if ((tokenSet_2_.member(cached_LA1))) { { match(tokenSet_2_); } } else { goto _loop44_breakloop; } } _loop44_breakloop :; } // ( ... )* _saveIndex = text.Length; match('\''); text.Length = _saveIndex; if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mLINEFOLDER(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = LINEFOLDER; mCRLF(false); { switch (cached_LA1) { case ' ': { mSPACE(false); break; } case '\t': { mHTAB(false); break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } } _ttype = Token.SKIP; if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mEMARK(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = EMARK; match('!'); { if ((cached_LA1 == '=')) { match('='); _ttype = BINOP; } else { } } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public static SourceLocation ToEndSourceLocation(antlr.IToken token) { return(new SourceLocation(token.getLine(), token.getColumn() + token.getText().Length - 1)); }
void ProcessNextNonWhiteSpaceToken(antlr.IToken token) { _lastNonWsToken = token; Enqueue(token); }
void Enqueue(antlr.IToken token) { _pendingTokens.Enqueue(token); }
public void mID(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; int _begin = text.Length; _ttype = ID; switch (cached_LA1) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { { switch (cached_LA1) { case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { matchRange('a', 'z'); break; } case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': { matchRange('A', 'Z'); break; } case '_': { match('_'); break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } } { // ( ... )* for (;;) { switch (cached_LA1) { case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { matchRange('a', 'z'); break; } case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': { matchRange('A', 'Z'); break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { matchRange('0', '9'); break; } case '_': { match('_'); break; } default: { goto _loop50_breakloop; } } } _loop50_breakloop :; } // ( ... )* break; } case '{': { { match('{'); { match(tokenSet_0_); } match('}'); } break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
protected void mSPECIAL(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = SPECIAL; switch (cached_LA1) { case '!': { match('\u0021'); break; } case '#': case '$': case '%': case '&': case '\'': case '(': case ')': case '*': case '+': { matchRange('\u0023', '\u002b'); break; } case '<': { match('\u003c'); break; } case '>': case '?': case '@': { matchRange('\u003e', '\u0040'); break; } case '[': { match('\u005b'); break; } case ']': case '^': { matchRange('\u005d', '\u005e'); break; } case '`': { match('\u0060'); break; } case '{': case '|': case '}': case '~': { matchRange('\u007b', '\u007e'); break; } default: if (((cached_LA1 >= '\u0080' && cached_LA1 <= '\u00ff'))) { matchRange('\u0080', '\u00ff'); } else { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } break; } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public override void initialize(IToken t) { base.initialize(t); line = t.getLine(); col = t.getColumn(); }
public static SourceLocation ToSourceLocation(antlr.IToken token) { return(new SourceLocation(token.getLine(), token.getColumn())); }
void EnqueueIndent(antlr.IToken prototype) { _pendingTokens.Enqueue(CreateToken(prototype, _indentTokenType, "<INDENT>")); }
private void SetEndSourceLocation(Node node, IToken token) { node.EndSourceLocation = SourceLocationFactory.ToSourceLocation(token); }
public MappedToken(ScanningLexer scanningLexer, antlr.IToken iToken) { lexer = scanningLexer; this.iToken = iToken; }
public void mIANA_TOKEN(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = IANA_TOKEN; { // ( ... )+ var _cnt82 = 0; for (;;) { switch (cached_LA1) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { mALPHA(false); break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { mDIGIT(false); break; } case '-': { mDASH(false); break; } case '_': { mUNDERSCORE(false); break; } default: if ((tokenSet_3_.member(cached_LA1))) { mSPECIAL(false); } else if (((cached_LA1 >= '\u0100' && cached_LA1 <= '\ufffe'))) { mUNICODE(false); } else { if (_cnt82 >= 1) { goto _loop82_breakloop; } else { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());; } } break; } _cnt82++; } _loop82_breakloop :; } // ( ... )+ var s = text.ToString(_begin, text.Length - _begin); int val; if (int.TryParse(s, out val)) { _ttype = NUMBER; } else { switch (s.ToUpper()) { case "BEGIN": _ttype = BEGIN; break; case "END": _ttype = END; break; case "VCALENDAR": _ttype = VCALENDAR; break; default: if (s.Length > 2 && s.Substring(0, 2).Equals("X-")) { _ttype = X_NAME; } break; } } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
public void mCTL(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = CTL; switch (cached_LA1) { case '\u0000': case '\u0001': case '\u0002': case '\u0003': case '\u0004': case '\u0005': case '\u0006': case '\u0007': case '\u0008': { matchRange('\u0000', '\u0008'); break; } case '\u000b': case '\u000c': case '\r': case '\u000e': case '\u000f': case '\u0010': case '\u0011': case '\u0012': case '\u0013': case '\u0014': case '\u0015': case '\u0016': case '\u0017': case '\u0018': case '\u0019': case '\u001a': case '\u001b': case '\u001c': case '\u001d': case '\u001e': case '\u001f': { matchRange('\u000b', '\u001F'); break; } case '\u007f': { match('\u007F'); break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
void EnqueueEOS(antlr.IToken prototype) { _pendingTokens.Enqueue(CreateToken(prototype, _eosTokenType, "<EOL>")); }
override public IToken nextToken() //throws TokenStreamException { IToken theRetToken = null; tryAgain: for (;;) { IToken _token = null; int _ttype = Token.INVALID_TYPE; resetText(); try // for char stream error handling { try // for lexical error handling { switch ( cached_LA1 ) { case '\t': case '\n': case '\r': case ' ': { mWS(true); theRetToken = returnToken_; break; } case '(': { mLPAREN(true); theRetToken = returnToken_; break; } case ')': { mRPAREN(true); theRetToken = returnToken_; break; } case '*': { mSTAR(true); theRetToken = returnToken_; break; } case '+': { mPLUS(true); theRetToken = returnToken_; break; } case ';': { mSEMI(true); theRetToken = returnToken_; break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { mINT(true); theRetToken = returnToken_; break; } default: { if (cached_LA1==EOF_CHAR) { uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE); } else {throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn());} } break; } if ( null==returnToken_ ) goto tryAgain; // found SKIP token _ttype = returnToken_.Type; _ttype = testLiteralsTable(_ttype); returnToken_.Type = _ttype; return returnToken_; } catch (RecognitionException e) { throw new TokenStreamRecognitionException(e); } } catch (CharStreamException cse) { if ( cse is CharStreamIOException ) { throw new TokenStreamIOException(((CharStreamIOException)cse).io); } else { throw new TokenStreamException(cse.Message); } } } }
public static LexicalInfo ToLexicalInfo(antlr.IToken token) { return(new LexicalInfo(token.getFilename(), token.getLine(), token.getColumn())); }
public ASTNode(IToken tok) : base(tok) { }
public static SourceLocation ToEndSourceLocation(antlr.IToken token) { string text = token.getText() ?? ""; return(new SourceLocation(token.getLine(), token.getColumn() + text.Length - 1)); }
override public IToken nextToken() //throws TokenStreamException { IToken theRetToken = null; tryAgain: for (;;) { IToken _token = null; int _ttype = Token.INVALID_TYPE; resetText(); try // for char stream error handling { try // for lexical error handling { switch (cached_LA1) { case ',': { mCOMMA(true); theRetToken = returnToken_; break; } case ';': { mSEMI(true); theRetToken = returnToken_; break; } case '(': { mLPAREN(true); theRetToken = returnToken_; break; } case ')': { mRPAREN(true); theRetToken = returnToken_; break; } case '?': { mQMARK(true); theRetToken = returnToken_; break; } case '!': { mEMARK(true); theRetToken = returnToken_; break; } case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '{': { mID(true); theRetToken = returnToken_; break; } case '&': case '=': case '^': case '|': { mBINOP(true); theRetToken = returnToken_; break; } case '0': case '1': { mCONST(true); theRetToken = returnToken_; break; } case '\t': case '\n': case '\r': case ' ': { mWS(true); theRetToken = returnToken_; break; } default: if ((cached_LA1 == ':') && (cached_LA2 == '=')) { mASSIGN(true); theRetToken = returnToken_; } else if ((cached_LA1 == ':') && (true)) { mCOLON(true); theRetToken = returnToken_; } else { if (cached_LA1 == EOF_CHAR) { uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE); } else { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } break; } if (null == returnToken_) { goto tryAgain; // found SKIP token } _ttype = returnToken_.Type; _ttype = testLiteralsTable(_ttype); returnToken_.Type = _ttype; return(returnToken_); } catch (RecognitionException e) { throw new TokenStreamRecognitionException(e); } } catch (CharStreamException cse) { if (cse is CharStreamIOException) { throw new TokenStreamIOException(((CharStreamIOException)cse).io); } else { throw new TokenStreamException(cse.Message); } } } }
override public IToken nextToken() //throws TokenStreamException { IToken theRetToken = null; tryAgain: for (;;) { IToken _token = null; var _ttype = Token.INVALID_TYPE; resetText(); try // for char stream error handling { try // for lexical error handling { switch (cached_LA1) { case '\n': { mLF(true); theRetToken = returnToken_; break; } case ' ': { mSPACE(true); theRetToken = returnToken_; break; } case '\t': { mHTAB(true); theRetToken = returnToken_; break; } case ':': { mCOLON(true); theRetToken = returnToken_; break; } case ';': { mSEMICOLON(true); theRetToken = returnToken_; break; } case ',': { mCOMMA(true); theRetToken = returnToken_; break; } case '.': { mDOT(true); theRetToken = returnToken_; break; } case '=': { mEQUAL(true); theRetToken = returnToken_; break; } case '/': { mSLASH(true); theRetToken = returnToken_; break; } case '"': { mDQUOTE(true); theRetToken = returnToken_; break; } default: if ((cached_LA1 == '\r') && (cached_LA2 == '\n') && (LA(3) == '\t' || LA(3) == ' ')) { mLINEFOLDER(true); theRetToken = returnToken_; } else if ((cached_LA1 == '\r') && (cached_LA2 == '\n') && (true)) { mCRLF(true); theRetToken = returnToken_; } else if ((cached_LA1 == '\\') && (tokenSet_0_.member(cached_LA2))) { mESCAPED_CHAR(true); theRetToken = returnToken_; } else if ((cached_LA1 == '\\') && (true)) { mBACKSLASH(true); theRetToken = returnToken_; } else if ((tokenSet_1_.member(cached_LA1)) && (true)) { mCTL(true); theRetToken = returnToken_; } else if ((tokenSet_2_.member(cached_LA1))) { mIANA_TOKEN(true); theRetToken = returnToken_; } else { if (cached_LA1 == EOF_CHAR) { uponEOF(); returnToken_ = makeToken(Token.EOF_TYPE); } else { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } break; } if (null == returnToken_) { goto tryAgain; // found SKIP token } _ttype = returnToken_.Type; _ttype = testLiteralsTable(_ttype); returnToken_.Type = _ttype; return(returnToken_); } catch (RecognitionException e) { throw new TokenStreamRecognitionException(e); } } catch (CharStreamException cse) { if (cse is CharStreamIOException) { throw new TokenStreamIOException(((CharStreamIOException)cse).io); } else { throw new TokenStreamException(cse.Message); } } } }
protected void mALPHA(bool _createToken) //throws RecognitionException, CharStreamException, TokenStreamException { int _ttype; IToken _token = null; var _begin = text.Length; _ttype = ALPHA; switch (cached_LA1) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': { matchRange('\u0041', '\u005a'); break; } case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': { matchRange('\u0061', '\u007a'); break; } default: { throw new NoViableAltForCharException(cached_LA1, getFilename(), getLine(), getColumn()); } } if (_createToken && (null == _token) && (_ttype != Token.SKIP)) { _token = makeToken(_ttype); _token.setText(text.ToString(_begin, text.Length - _begin)); } returnToken_ = _token; }
private LexicalInfo ToLexicalInfo(IToken token) { return SourceLocationFactory.ToLexicalInfo(token); }
private MemberReferenceExpression MemberReferenceForToken(Expression target, IToken memberName) { MemberReferenceExpression mre = new MemberReferenceExpression(ToLexicalInfo(memberName)); mre.Target = target; mre.Name = memberName.getText(); return mre; }