protected internal virtual int ExecATN(ICharStream input, DFAState ds0) { //System.out.println("enter exec index "+input.index()+" from "+ds0.configs); int t = input.La(1); DFAState s = ds0; // s is current/from DFA state while (true) { // while more work // As we move src->trg, src->trg, we keep track of the previous trg to // avoid looking up the DFA state again, which is expensive. // If the previous target was already part of the DFA, we might // be able to avoid doing a reach operation upon t. If s!=null, // it means that semantic predicates didn't prevent us from // creating a DFA state. Once we know s!=null, we check to see if // the DFA state has an edge already for t. If so, we can just reuse // it's configuration set; there's no point in re-computing it. // This is kind of like doing DFA simulation within the ATN // simulation because DFA simulation is really just a way to avoid // computing reach/closure sets. Technically, once we know that // we have a previously added DFA state, we could jump over to // the DFA simulator. But, that would mean popping back and forth // a lot and making things more complicated algorithmically. // This optimization makes a lot of sense for loops within DFA. // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. DFAState target = GetExistingTargetState(s, t); if (target == null) { target = ComputeTargetState(input, s, t); } if (target == Error) { break; } if (target.isAcceptState) { CaptureSimState(prevAccept, input, target); if (t == IntStreamConstants.Eof) { break; } } if (t != IntStreamConstants.Eof) { Consume(input); t = input.La(1); } s = target; } // flip; current DFA target becomes new src/from state return(FailOrAccept(prevAccept, input, s.configs, t)); }
public JavaUnicodeInputStream([NotNull] ICharStream source) { if (source == null) { throw new ArgumentNullException("source"); } this._source = source; this._la1 = source.La(1); }
public static int FindChar(int start, int stop, ICharStream input, char c) { for (int i = start - input.Index + 1; i <= stop - input.Index; i++) { if (input.La(i) == c) { return(input.Index + i); } } return(-1); }
public static bool ConsumeMlComment(this ICharStream stream, IPairFactory pairFactory, Pair parent) { if (stream.Next != '\"') { return(false); } if (stream.La(2) != '\"') { return(false); } if (stream.La(3) != '\"') { return(false); } stream.Consume(); var begin = new CharLocation(stream); stream.Consume(); stream.Consume(); while (!(stream.Next == '\"' && stream.La(2) == '\"' && stream.La(3) == '\"') && stream.Next != -1) { stream.Consume(); } if (stream.Next == '\"') { stream.Consume(); stream.Consume(); stream.Consume(); } var comment = pairFactory.ProcessComment((ITextSource)stream, 2, new Interval(begin, new CharLocation(stream))); if (comment != null) { pairFactory.AppendChild(parent, comment); } return(true); }
public static bool ConsumeSlComment(this ICharStream stream, IPairFactory pairFactory, Pair parent) { if (stream.Next != '\'') { return(false); } if (stream.La(2) != '\'') { return(false); } if (stream.La(3) != '\'') { return(false); } stream.Consume(); var begin = new CharLocation(stream); stream.Consume(); stream.Consume(); var c = stream.Next; while (!c.IsNewLineCharacter() && c != -1) { stream.Consume(); c = stream.Next; } var comment = pairFactory.ProcessComment((ITextSource)stream, 1, new Interval(begin, new CharLocation(stream))); if (comment != null) { pairFactory.AppendChild(parent, comment); } return(true); }
protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos) { // seek to after last char in token input.Seek(index); this.line = line; this.charPositionInLine = charPos; if (input.La(1) != IntStreamConstants.Eof) { Consume(input); } if (lexerActionExecutor != null && recog != null) { lexerActionExecutor.Execute(recog, input, startIndex); } }
public virtual void Consume(ICharStream input) { int curChar = input.La(1); if (curChar == '\n') { _line++; charPositionInLine = 0; } else { charPositionInLine++; } input.Consume(); }
protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex , int index, int line, int charPos) { if (actionIndex >= 0 && recog != null) { recog.Action(null, ruleIndex, actionIndex); } // seek to after last char in token input.Seek(index); this.line = line; this.charPositionInLine = charPos; if (input.La(1) != IntStreamConstants.Eof) { Consume(input); } }
public int La(int i) { int c = stream.La(i); if (c \ <= 0) { return(c); } char o = (char)c; if (upper) { return((int)char.ToUpperInvariant(o)); } return((int)char.ToLowerInvariant(o)); }
public int La(int i) { int c = stream.La(i); if (c <= 0) { return(c); } char o = (char)c; if (convertFromNonUS) { if (o == ';') { return((int)','); } if (o == ',') { return((int)'.'); } } return((int)char.ToUpperInvariant(o)); }
public static int LA(this ICharStream self, int i) { return(self.La(i: i)); }
protected internal virtual int ExecATN(ICharStream input, DFAState ds0) { //System.out.println("enter exec index "+input.index()+" from "+ds0.configs); int t = input.La(1); DFAState s = ds0; // s is current/from DFA state while (true) { // while more work // As we move src->trg, src->trg, we keep track of the previous trg to // avoid looking up the DFA state again, which is expensive. // If the previous target was already part of the DFA, we might // be able to avoid doing a reach operation upon t. If s!=null, // it means that semantic predicates didn't prevent us from // creating a DFA state. Once we know s!=null, we check to see if // the DFA state has an edge already for t. If so, we can just reuse // it's configuration set; there's no point in re-computing it. // This is kind of like doing DFA simulation within the ATN // simulation because DFA simulation is really just a way to avoid // computing reach/closure sets. Technically, once we know that // we have a previously added DFA state, we could jump over to // the DFA simulator. But, that would mean popping back and forth // a lot and making things more complicated algorithmically. // This optimization makes a lot of sense for loops within DFA. // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. DFAState target = GetExistingTargetState(s, t); if (target == null) { target = ComputeTargetState(input, s, t); } if (target == Error) { break; } if (target.isAcceptState) { CaptureSimState(prevAccept, input, target); if (t == IntStreamConstants.Eof) { break; } } if (t != IntStreamConstants.Eof) { Consume(input); t = input.La(1); } s = target; } // flip; current DFA target becomes new src/from state return FailOrAccept(prevAccept, input, s.configs, t); }
public static int LA(this ICharStream stream, int i) { return(stream.La(i)); }
/// <summary> /// Consumes indent of the multiline string line. /// Returns true if indent is greater than current indent (ml string continues) /// </summary> /// <returns></returns> private void ParseMlStringIndent() { var currentPair = _lineState.CurrentPair; var indent = GetIndent(_lineState.Indent, currentPair, out var endedByComment); if (_input.Next != -1 && indent > _lineState.Indent && // Indent is greater than current indent (ml string continues). !endedByComment //Dedent in comments ends ML string ) { ParseMlValue(); return; } // At this point we know that ml string is ended. var valueStart = currentPair.ValueQuotesType; if (valueStart > 0) { //Quoted string ended by indentation (missing quote) currentPair.MissingValueQuote = true; //newPair = AppendCurrentPair(); ReportMLSSyntaxError(1, new Interval(currentPair.ValueInterval.End, currentPair.ValueInterval.End), valueStart); } else { //Multiline Open string if (indent == _lineState.Indent && _input.Next == '=' && _input.La(2) == '=' && _input.La(3) == '=') { //Checking if it is ended with === _input.Consume(); _input.Consume(); _input.Consume(); //Consuming === currentPair.ValueInterval = new Interval(currentPair.ValueInterval.Begin, new CharLocation(_input)); } } var newPair = AppendCurrentPair(); //Creating the new pair and appending it to parent //Reporting end of pair if (_input.Next != -1) { _pairFactory.EndPair(newPair, new Interval(GetPairEnd((IMappedPair)newPair))); } else { _pairFactory.EndPair(newPair, indent <= _lineState.Indent ? new Interval(GetPairEnd((IMappedPair)newPair)) : new Interval(_input), _lineState.State == ParserStateEnum.Value || indent > _lineState.Indent); //Special case used in completion. Value context. True- means value is ended by EOF but not by dedent. } _lineState.Indent = indent; //Saving new value of indent in the lineState //Ending pair with bigger indents. while (_pairStack.Peek().Indent >= indent) { EndPair(new Interval(_input)); } //Checking for BlockIndentationMismatch error if (_input.Next != -1 && //ignore indent mismatch in the EOF _pairStack.Peek().BlockIndent != indent) { ReportBlockIndentationMismatch(new Interval(new CharLocation(_input.Line, 1, _input.Index - indent), new CharLocation(_input))); } _lineState.State = ParserStateEnum.PairDelimiter; }
public void Consume() { if (_la1 != '\\') { _source.Consume(); _la1 = _source.La(1); _range = Math.Max(_range, _source.Index); _slashCount = 0; return; } // make sure the next character has been processed this.La(1); if (_escapeListIndex >= _escapeIndexes.Count || _escapeIndexes[_escapeListIndex] != Index) { _source.Consume(); _slashCount++; } else { int indirectionLevel = _escapeIndirectionLevels[_escapeListIndex]; for (int i = 0; i < 6 + indirectionLevel; i++) { _source.Consume(); } _escapeListIndex++; _slashCount = 0; } _la1 = _source.La(1); Debug.Assert(_range >= Index); }
public virtual void Consume(ICharStream input) { int curChar = input.La(1); if (curChar == '\n') { line++; charPositionInLine = 0; } else { charPositionInLine++; } input.Consume(); }
public static void Check_CobolCharStream() { // Test file properties string relativePath = @"Compiler\Parser\Samples"; string textName = "MSVCOUT"; DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat; // Compile test file CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat); // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, compilationDocument.TokensDocumentSnapshot.Lines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, tokensIterator); tokenSource.NextToken(); // Get underlying CharStream ICharStream charStream = tokenSource.InputStream; if (charStream.Index != 0) { throw new Exception("Char stream index should start at 0"); } if (charStream.La(0) != 0) { throw new Exception("La(0) should be 0"); } if (charStream.La(1) != '0') { throw new Exception("La(1) should be 0"); } if (charStream.La(4) != '1') { throw new Exception("La(4) should be 1"); } if (charStream.La(5) != '6') { throw new Exception("La(5) should be 6"); } charStream.Consume(); if (charStream.Index != 1) { throw new Exception("Char stream index should be 1 after consume"); } if (charStream.La(4) != '6') { throw new Exception("La(4) should be 6 after consume"); } if (charStream.La(80) != IntStreamConstants.Eof) { throw new Exception("La(80) should be Eof"); } charStream.Seek(12); if (charStream.Index != 12) { throw new Exception("Char stream index should be 12 after seek"); } if (charStream.La(-1) != ':') { throw new Exception("La(-1) should be : after seek"); } if (charStream.La(1) != 'M') { throw new Exception("La(1) should be M after seek"); } // should do nothing int marker = charStream.Mark(); charStream.Release(marker); if (charStream.La(2) != 'S') { throw new Exception("La(2) should be S after release"); } string text = charStream.GetText(new Interval(11, 18)); if (text != ":MSVCOUT") { throw new Exception("Char stream GetText method KO"); } if (charStream.Size != 80) { throw new Exception("Char stream size KO"); } }
/// <summary> /// Return a token from this source; i.e., match a token on the char /// stream. /// </summary> /// <remarks> /// Return a token from this source; i.e., match a token on the char /// stream. /// </remarks> public virtual IToken NextToken() { if (_input == null) { throw new InvalidOperationException("nextToken requires a non-null input stream."); } // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.Mark(); try { while (true) { if (_hitEOF) { EmitEOF(); return(_token); } _token = null; _channel = TokenConstants.DefaultChannel; _tokenStartCharIndex = _input.Index; _tokenStartCharPositionInLine = Interpreter.Column; _tokenStartLine = Interpreter.Line; _text = null; do { _type = TokenConstants.InvalidType; // System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+ // " in mode "+mode+ // " at index "+input.index()); int ttype; try { ttype = Interpreter.Match(_input, _mode); } catch (LexerNoViableAltException e) { NotifyListeners(e); // report error Recover(e); ttype = TokenTypes.Skip; } if (_input.La(1) == IntStreamConstants.Eof) { _hitEOF = true; } if (_type == TokenConstants.InvalidType) { _type = ttype; } if (_type == TokenTypes.Skip) { goto outer_continue; } }while (_type == TokenTypes.More); if (_token == null) { Emit(); } return(_token); outer_continue :; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.Release(tokenStartMarker); } }
protected internal virtual int ExecATN(ICharStream input, DFAState ds0) { //System.out.println("enter exec index "+input.index()+" from "+ds0.configs); int t = input.La(1); DFAState s = ds0; // s is current/from DFA state while (true) { // while more work // As we move src->trg, src->trg, we keep track of the previous trg to // avoid looking up the DFA state again, which is expensive. // If the previous target was already part of the DFA, we might // be able to avoid doing a reach operation upon t. If s!=null, // it means that semantic predicates didn't prevent us from // creating a DFA state. Once we know s!=null, we check to see if // the DFA state has an edge already for t. If so, we can just reuse // it's configuration set; there's no point in re-computing it. // This is kind of like doing DFA simulation within the ATN // simulation because DFA simulation is really just a way to avoid // computing reach/closure sets. Technically, once we know that // we have a previously added DFA state, we could jump over to // the DFA simulator. But, that would mean popping back and forth // a lot and making things more complicated algorithmically. // This optimization makes a lot of sense for loops within DFA. // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. ATNConfigSet closure = s.configs; DFAState target = null; target = s.GetTarget(t); if (target == Error) { break; } #if !PORTABLE if (debug && target != null) { System.Console.Out.WriteLine("reuse state " + s.stateNumber + " edge to " + target .stateNumber); } #endif if (target == null) { ATNConfigSet reach = new OrderedATNConfigSet(); // if we don't find an existing DFA state // Fill reach starting from closure, following t transitions GetReachableConfigSet(input, closure, reach, t); if (reach.Count == 0) { // we got nowhere on t from s // we reached state associated with closure for sure, so // make sure it's defined. worst case, we define s0 from // start state configs. DFAState from = s != null ? s : AddDFAState(closure); // we got nowhere on t, don't throw out this knowledge; it'd // cause a failover from DFA later. AddDFAEdge(from, t, Error); break; } // stop when we can't match any more char // Add an edge from s to target DFA found/created for reach target = AddDFAEdge(s, t, reach); } if (target.isAcceptState) { CaptureSimState(prevAccept, input, target); if (t == IntStreamConstants.Eof) { break; } } if (t != IntStreamConstants.Eof) { Consume(input); t = input.La(1); } s = target; } // flip; current DFA target becomes new src/from state return FailOrAccept(prevAccept, input, s.configs, t); }