public int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFA dfa = decisionToDFA[mode]; if (dfa.s0 == null) { return(MatchATN(input)); } else { return(ExecATN(input, dfa.s0)); } } finally { input.Release(mark); } }
public virtual int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFAState s0 = atn.modeToDFA[mode].s0.Get(); if (s0 == null) { return(MatchATN(input)); } else { return(ExecATN(input, s0)); } } finally { input.Release(mark); } }
/** * Evaluate a predicate specified in the lexer. * * <p>If {@code speculative} is {@code true}, this method was called before * {@link #consume} for the matched character. This method should call * {@link #consume} before evaluating the predicate to ensure position * sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine}, * and {@link Lexer#getCharPositionInLine}, properly reflect the current * lexer state. This method should restore {@code input} and the simulator * to the original state before returning (i.e. undo the actions made by the * call to {@link #consume}.</p> * * @param input The input stream. * @param ruleIndex The rule containing the predicate. * @param predIndex The index of the predicate within the rule. * @param speculative {@code true} if the current index in {@code input} is * one character before the predicate's location. * * @return {@code true} if the specified predicate evaluates to * {@code true}. */ protected bool EvaluatePredicate(ICharStream input, int ruleIndex, int predIndex, bool speculative) { // assume true if no recognizer was provided if (recog == null) { return(true); } if (!speculative) { return(recog.Sempred(null, ruleIndex, predIndex)); } int savedCharPositionInLine = charPositionInLine; int savedLine = thisLine; int index = input.Index; int marker = input.Mark(); try { Consume(input); return(recog.Sempred(null, ruleIndex, predIndex)); } finally { charPositionInLine = savedCharPositionInLine; thisLine = savedLine; input.Seek(index); input.Release(marker); } }
private TemplateLexerMode CheckAnonymousTemplateForParameters() { int position = _input.Mark(); ClassifierLexerState currentState = State; try { Mode = TemplateLexerMode.AnonymousTemplateParameters; bool previousWasArg = false; while (true) { IToken token = NextToken(); switch (token.Type) { case OutsideClassifierLexer.COMMA: if (!previousWasArg) { return(TemplateLexerMode.Template); } previousWasArg = false; continue; case OutsideClassifierLexer.PARAMETER_DEFINITION: case OutsideClassifierLexer.ID: if (previousWasArg) { return(TemplateLexerMode.Template); } previousWasArg = true; continue; case OutsideClassifierLexer.PIPE: if (previousWasArg) { return(TemplateLexerMode.AnonymousTemplateParameters); } return(TemplateLexerMode.Template); case OutsideClassifierLexer.WS: case OutsideClassifierLexer.COMMENT: case OutsideClassifierLexer.NEWLINE: continue; default: return(TemplateLexerMode.Template); } } } finally { _input.Rewind(position); State = currentState; } }
private IToken subTemplate() { // look for "{ args ID (',' ID)* '|' ..." subtemplateDepth++; int m = input.Mark(); int curlyStartChar = startCharIndex; int curlyLine = startLine; int curlyPos = startCharPositionInLine; List <IToken> argTokens = new List <IToken>(); consume(); IToken curly = newTokenFromPreviousChar(LCURLY); WS(); argTokens.Add(mID()); WS(); while (c == ',') { consume(); argTokens.Add(newTokenFromPreviousChar(COMMA)); WS(); argTokens.Add(mID()); WS(); } WS(); if (c == '|') { consume(); argTokens.Add(newTokenFromPreviousChar(PIPE)); if (isWS(c)) { consume(); // ignore a single whitespace after | } //System.out.println("matched args: "+argTokens); foreach (IToken t in argTokens) { emit(t); } input.Release(m); scanningInsideExpr = false; startCharIndex = curlyStartChar; // reset state startLine = curlyLine; startCharPositionInLine = curlyPos; return(curly); } input.Rewind(m); startCharIndex = curlyStartChar; // reset state startLine = curlyLine; startCharPositionInLine = curlyPos; consume(); scanningInsideExpr = false; return(curly); }
/// <summary> /// Return a token from this source; i.e., match a token on the char /// stream. /// </summary> /// <remarks> /// Return a token from this source; i.e., match a token on the char /// stream. /// </remarks> public virtual IToken NextToken() { if (_input == null) { throw new InvalidOperationException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.Mark(); try { while (true) { if (_hitEOF) { EmitEOF(); return(_token); } _token = null; _channel = TokenConstants.DefaultChannel; _tokenStartCharIndex = _input.Index; _tokenStartColumn = Interpreter.Column; _tokenStartLine = Interpreter.Line; _text = null; do { _type = TokenConstants.InvalidType; // System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+ // " in mode "+mode+ // " at index "+input.index()); int ttype; try { ttype = Interpreter.Match(_input, _mode); } catch (LexerNoViableAltException e) { NotifyListeners(e); // report error Recover(e); ttype = TokenTypes.Skip; } if (_input.LA(1) == IntStreamConstants.EOF) { _hitEOF = true; } if (_type == TokenConstants.InvalidType) { _type = ttype; } if (_type == TokenTypes.Skip) { goto outer_continue; } }while (_type == TokenTypes.More); if (_token == null) { Emit(); } return(_token); outer_continue :; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.Release(tokenStartMarker); } }
//public int La(int i) // } public int Mark() { return(stream.Mark()); }
private IToken NextTokenCore() { IToken token = null; switch (Mode) { case AntlrClassifierLexerMode.Action: if (ActionLevel == 1 && (InOptions || InTokens) && _input.LA(1) != '}') { goto case AntlrClassifierLexerMode.Grammar; } switch (_input.LA(1)) { case '{': token = _grammarLexer.NextToken(); ActionLevel++; token.Type = AntlrGrammarClassifierLexer.ACTION; break; case '}': token = _grammarLexer.NextToken(); ActionLevel--; token.Type = AntlrGrammarClassifierLexer.ACTION; if (ActionLevel == 0) { Mode = AntlrClassifierLexerMode.Grammar; if (InOptions || InTokens) { token.Type = AntlrGrammarClassifierLexer.RCURLY; InOptions = false; InTokens = false; } } break; default: token = _actionLexer.NextToken(); break; } break; case AntlrClassifierLexerMode.ActionCharLiteral: case AntlrClassifierLexerMode.ActionStringLiteral: case AntlrClassifierLexerMode.ArgActionCharLiteral: case AntlrClassifierLexerMode.ArgActionStringLiteral: token = _actionLexer.NextToken(); break; case AntlrClassifierLexerMode.ArgAction: if (_input.LA(1) == ']') { token = _grammarLexer.NextToken(); Mode = AntlrClassifierLexerMode.Grammar; } else { token = _actionLexer.NextToken(); } break; case AntlrClassifierLexerMode.GrammarDoubleAngleStringLiteral: token = _grammarLexer.NextToken(); break; case AntlrClassifierLexerMode.Grammar: default: token = _grammarLexer.NextToken(); switch (token.Type) { case AntlrGrammarClassifierLexer.LCURLY: ActionLevel++; Mode = AntlrClassifierLexerMode.Action; if ((!InOptions && !InTokens) || ActionLevel != 1) { token.Type = AntlrGrammarClassifierLexer.ACTION; } break; case AntlrGrammarClassifierLexer.LBRACK: Mode = AntlrClassifierLexerMode.ArgAction; break; case AntlrGrammarClassifierLexer.IDENTIFIER: switch (token.Text) { case "options": InOptions = true; break; case "tokens": InTokens = true; break; default: if (InOptions) { token.Type = AntlrGrammarClassifierLexer.OptionValue; } AntlrClassifierLexerState currentState = GetCurrentState(); int marker = _input.Mark(); try { while (true) { IToken nextToken = NextToken(); switch (nextToken.Type) { case AntlrGrammarClassifierLexer.NEWLINE: case AntlrGrammarClassifierLexer.WS: case AntlrGrammarClassifierLexer.COMMENT: case AntlrGrammarClassifierLexer.DOC_COMMENT: case AntlrGrammarClassifierLexer.ML_COMMENT: case AntlrGrammarClassifierLexer.SL_COMMENT: continue; case AntlrGrammarClassifierLexer.ASSIGN: if (InOptions) { if (IsValidOption(token.Text)) { token.Type = AntlrGrammarClassifierLexer.ValidGrammarOption; } else { token.Type = AntlrGrammarClassifierLexer.InvalidGrammarOption; } } else if (InTokens) { } else { token.Type = AntlrGrammarClassifierLexer.LABEL; } break; case AntlrGrammarClassifierLexer.PLUS_ASSIGN: token.Type = AntlrGrammarClassifierLexer.LABEL; break; default: break; } break; } } finally { _input.Rewind(marker); SetCurrentState(currentState); } break; } break; default: break; } break; } return(token); }
/// <summary>Evaluate a predicate specified in the lexer.</summary> /// <remarks> /// Evaluate a predicate specified in the lexer. /// <p/> /// If /// <code>speculative</code> /// is /// <code>true</code> /// , this method was called before /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// for the matched character. This method should call /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// before evaluating the predicate to ensure position /// sensitive values, including /// <see cref="Antlr4.Runtime.Lexer.Text()">Antlr4.Runtime.Lexer.Text()</see> /// , /// <see cref="Antlr4.Runtime.Lexer.Line()">Antlr4.Runtime.Lexer.Line()</see> /// , /// and /// <see cref="Antlr4.Runtime.Lexer.Column()">Antlr4.Runtime.Lexer.Column()</see> /// , properly reflect the current /// lexer state. This method should restore /// <code>input</code> /// and the simulator /// to the original state before returning (i.e. undo the actions made by the /// call to /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// . /// </remarks> /// <param name="input">The input stream.</param> /// <param name="ruleIndex">The rule containing the predicate.</param> /// <param name="predIndex">The index of the predicate within the rule.</param> /// <param name="speculative"> /// /// <code>true</code> /// if the current index in /// <code>input</code> /// is /// one character before the predicate's location. /// </param> /// <returns> /// /// <code>true</code> /// if the specified predicate evaluates to /// <code>true</code> /// . /// </returns> protected internal virtual bool EvaluatePredicate(ICharStream input, int ruleIndex , int predIndex, bool speculative) { // assume true if no recognizer was provided if (recog == null) { return true; } if (!speculative) { return recog.Sempred(null, ruleIndex, predIndex); } int savedCharPositionInLine = charPositionInLine; int savedLine = line; int index = input.Index; int marker = input.Mark(); try { Consume(input); return recog.Sempred(null, ruleIndex, predIndex); } finally { charPositionInLine = savedCharPositionInLine; line = savedLine; input.Seek(index); input.Release(marker); } }
public virtual int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFAState s0 = atn.modeToDFA[mode].s0.Get(); if (s0 == null) { return MatchATN(input); } else { return ExecATN(input, s0); } } finally { input.Release(mark); } }
public static void Check_CobolCharStream() { // Test file properties string relativePath = @"Compiler\Parser\Samples"; string textName = "MSVCOUT"; DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat; // Compile test file CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat); // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, compilationDocument.TokensDocumentSnapshot.Lines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, tokensIterator); tokenSource.NextToken(); // Get underlying CharStream ICharStream charStream = tokenSource.InputStream; if (charStream.Index != 0) { throw new Exception("Char stream index should start at 0"); } if (charStream.La(0) != 0) { throw new Exception("La(0) should be 0"); } if (charStream.La(1) != '0') { throw new Exception("La(1) should be 0"); } if (charStream.La(4) != '1') { throw new Exception("La(4) should be 1"); } if (charStream.La(5) != '6') { throw new Exception("La(5) should be 6"); } charStream.Consume(); if (charStream.Index != 1) { throw new Exception("Char stream index should be 1 after consume"); } if (charStream.La(4) != '6') { throw new Exception("La(4) should be 6 after consume"); } if (charStream.La(80) != IntStreamConstants.Eof) { throw new Exception("La(80) should be Eof"); } charStream.Seek(12); if (charStream.Index != 12) { throw new Exception("Char stream index should be 12 after seek"); } if (charStream.La(-1) != ':') { throw new Exception("La(-1) should be : after seek"); } if (charStream.La(1) != 'M') { throw new Exception("La(1) should be M after seek"); } // should do nothing int marker = charStream.Mark(); charStream.Release(marker); if (charStream.La(2) != 'S') { throw new Exception("La(2) should be S after release"); } string text = charStream.GetText(new Interval(11, 18)); if (text != ":MSVCOUT") { throw new Exception("Char stream GetText method KO"); } if (charStream.Size != 80) { throw new Exception("Char stream size KO"); } }
public int Mark() => internalStream.Mark();
public int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFA dfa = decisionToDFA[mode]; if (dfa.s0 == null) { return MatchATN(input); } else { return ExecATN(input, dfa.s0); } } finally { input.Release(mark); } }
public int Mark() { return(_source.Mark()); }