/** * Evaluate a predicate specified in the lexer. * * <p>If {@code speculative} is {@code true}, this method was called before * {@link #consume} for the matched character. This method should call * {@link #consume} before evaluating the predicate to ensure position * sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine}, * and {@link Lexer#getCharPositionInLine}, properly reflect the current * lexer state. This method should restore {@code input} and the simulator * to the original state before returning (i.e. undo the actions made by the * call to {@link #consume}.</p> * * @param input The input stream. * @param ruleIndex The rule containing the predicate. * @param predIndex The index of the predicate within the rule. * @param speculative {@code true} if the current index in {@code input} is * one character before the predicate's location. * * @return {@code true} if the specified predicate evaluates to * {@code true}. */ protected bool EvaluatePredicate(ICharStream input, int ruleIndex, int predIndex, bool speculative) { // assume true if no recognizer was provided if (recog == null) { return(true); } if (!speculative) { return(recog.Sempred(null, ruleIndex, predIndex)); } int savedCharPositionInLine = charPositionInLine; int savedLine = thisLine; int index = input.Index; int marker = input.Mark(); try { Consume(input); return(recog.Sempred(null, ruleIndex, predIndex)); } finally { charPositionInLine = savedCharPositionInLine; thisLine = savedLine; input.Seek(index); input.Release(marker); } }
public int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFA dfa = decisionToDFA[mode]; if (dfa.s0 == null) { return(MatchATN(input)); } else { return(ExecATN(input, dfa.s0)); } } finally { input.Release(mark); } }
public virtual int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFAState s0 = atn.modeToDFA[mode].s0.Get(); if (s0 == null) { return(MatchATN(input)); } else { return(ExecATN(input, s0)); } } finally { input.Release(mark); } }
private IToken subTemplate() { // look for "{ args ID (',' ID)* '|' ..." subtemplateDepth++; int m = input.Mark(); int curlyStartChar = startCharIndex; int curlyLine = startLine; int curlyPos = startCharPositionInLine; List <IToken> argTokens = new List <IToken>(); consume(); IToken curly = newTokenFromPreviousChar(LCURLY); WS(); argTokens.Add(mID()); WS(); while (c == ',') { consume(); argTokens.Add(newTokenFromPreviousChar(COMMA)); WS(); argTokens.Add(mID()); WS(); } WS(); if (c == '|') { consume(); argTokens.Add(newTokenFromPreviousChar(PIPE)); if (isWS(c)) { consume(); // ignore a single whitespace after | } //System.out.println("matched args: "+argTokens); foreach (IToken t in argTokens) { emit(t); } input.Release(m); scanningInsideExpr = false; startCharIndex = curlyStartChar; // reset state startLine = curlyLine; startCharPositionInLine = curlyPos; return(curly); } input.Rewind(m); startCharIndex = curlyStartChar; // reset state startLine = curlyLine; startCharPositionInLine = curlyPos; consume(); scanningInsideExpr = false; return(curly); }
static void Main(string[] args) { var options = lfmt.FormatOptions.Parse(args); if (options == null) { return; } ICharStream charStream = null; switch (options.inputMode) { case "text": charStream = CharStreams.fromstring(options.input); break; case "file": charStream = CharStreams.fromPath(options.input); break; case "stdin": charStream = CharStreams.fromTextReader(Console.In); break; default: // show error messages return; } IFormatWriter writer = null; switch (options.outputMode) { case "stdout": writer = new StdoutWriter(); break; case "file": writer = new FileWriter(options.output); break; default: // show error messages return; } lfmt.Formatter.Format(charStream, writer, options); charStream.Release(0); writer.Close(); }
/// <summary> /// Return a token from this source; i.e., match a token on the char /// stream. /// </summary> /// <remarks> /// Return a token from this source; i.e., match a token on the char /// stream. /// </remarks> public virtual IToken NextToken() { if (_input == null) { throw new InvalidOperationException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.Mark(); try { while (true) { if (_hitEOF) { EmitEOF(); return(_token); } _token = null; _channel = TokenConstants.DefaultChannel; _tokenStartCharIndex = _input.Index; _tokenStartColumn = Interpreter.Column; _tokenStartLine = Interpreter.Line; _text = null; do { _type = TokenConstants.InvalidType; // System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+ // " in mode "+mode+ // " at index "+input.index()); int ttype; try { ttype = Interpreter.Match(_input, _mode); } catch (LexerNoViableAltException e) { NotifyListeners(e); // report error Recover(e); ttype = TokenTypes.Skip; } if (_input.LA(1) == IntStreamConstants.EOF) { _hitEOF = true; } if (_type == TokenConstants.InvalidType) { _type = ttype; } if (_type == TokenTypes.Skip) { goto outer_continue; } }while (_type == TokenTypes.More); if (_token == null) { Emit(); } return(_token); outer_continue :; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.Release(tokenStartMarker); } }
public void Release(int marker) { stream.Release(marker); }
/// <summary>Evaluate a predicate specified in the lexer.</summary> /// <remarks> /// Evaluate a predicate specified in the lexer. /// <p/> /// If /// <code>speculative</code> /// is /// <code>true</code> /// , this method was called before /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// for the matched character. This method should call /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// before evaluating the predicate to ensure position /// sensitive values, including /// <see cref="Antlr4.Runtime.Lexer.Text()">Antlr4.Runtime.Lexer.Text()</see> /// , /// <see cref="Antlr4.Runtime.Lexer.Line()">Antlr4.Runtime.Lexer.Line()</see> /// , /// and /// <see cref="Antlr4.Runtime.Lexer.Column()">Antlr4.Runtime.Lexer.Column()</see> /// , properly reflect the current /// lexer state. This method should restore /// <code>input</code> /// and the simulator /// to the original state before returning (i.e. undo the actions made by the /// call to /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream) /// </see> /// . /// </remarks> /// <param name="input">The input stream.</param> /// <param name="ruleIndex">The rule containing the predicate.</param> /// <param name="predIndex">The index of the predicate within the rule.</param> /// <param name="speculative"> /// /// <code>true</code> /// if the current index in /// <code>input</code> /// is /// one character before the predicate's location. /// </param> /// <returns> /// /// <code>true</code> /// if the specified predicate evaluates to /// <code>true</code> /// . /// </returns> protected internal virtual bool EvaluatePredicate(ICharStream input, int ruleIndex , int predIndex, bool speculative) { // assume true if no recognizer was provided if (recog == null) { return true; } if (!speculative) { return recog.Sempred(null, ruleIndex, predIndex); } int savedCharPositionInLine = charPositionInLine; int savedLine = line; int index = input.Index; int marker = input.Mark(); try { Consume(input); return recog.Sempred(null, ruleIndex, predIndex); } finally { charPositionInLine = savedCharPositionInLine; line = savedLine; input.Seek(index); input.Release(marker); } }
public virtual int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFAState s0 = atn.modeToDFA[mode].s0.Get(); if (s0 == null) { return MatchATN(input); } else { return ExecATN(input, s0); } } finally { input.Release(mark); } }
public static void Check_CobolCharStream() { // Test file properties string relativePath = @"Compiler\Parser\Samples"; string textName = "MSVCOUT"; DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat; // Compile test file CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat); // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, compilationDocument.TokensDocumentSnapshot.Lines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, tokensIterator); tokenSource.NextToken(); // Get underlying CharStream ICharStream charStream = tokenSource.InputStream; if (charStream.Index != 0) { throw new Exception("Char stream index should start at 0"); } if (charStream.La(0) != 0) { throw new Exception("La(0) should be 0"); } if (charStream.La(1) != '0') { throw new Exception("La(1) should be 0"); } if (charStream.La(4) != '1') { throw new Exception("La(4) should be 1"); } if (charStream.La(5) != '6') { throw new Exception("La(5) should be 6"); } charStream.Consume(); if (charStream.Index != 1) { throw new Exception("Char stream index should be 1 after consume"); } if (charStream.La(4) != '6') { throw new Exception("La(4) should be 6 after consume"); } if (charStream.La(80) != IntStreamConstants.Eof) { throw new Exception("La(80) should be Eof"); } charStream.Seek(12); if (charStream.Index != 12) { throw new Exception("Char stream index should be 12 after seek"); } if (charStream.La(-1) != ':') { throw new Exception("La(-1) should be : after seek"); } if (charStream.La(1) != 'M') { throw new Exception("La(1) should be M after seek"); } // should do nothing int marker = charStream.Mark(); charStream.Release(marker); if (charStream.La(2) != 'S') { throw new Exception("La(2) should be S after release"); } string text = charStream.GetText(new Interval(11, 18)); if (text != ":MSVCOUT") { throw new Exception("Char stream GetText method KO"); } if (charStream.Size != 80) { throw new Exception("Char stream size KO"); } }
public void Release(int marker) => internalStream.Release(marker);
public int Match(ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFA dfa = decisionToDFA[mode]; if (dfa.s0 == null) { return MatchATN(input); } else { return ExecATN(input, dfa.s0); } } finally { input.Release(mark); } }
public void Release(int marker) { _source.Release(marker); }