/// <summary> /// generate from a input string a rule by parsing and compiling and store it /// </summary> /// <param name="input"></param> /// <returns></returns> private bool Generate(Antlr4.Runtime.ICharStream input) { try { RulezLexer aLexer = new RulezLexer(input); // wrap a token-stream around the lexer Antlr4.Runtime.CommonTokenStream theTokens = new Antlr4.Runtime.CommonTokenStream(aLexer); // create the aParser RulezParser aParser = new RulezParser(theTokens); aParser.Trace = true; aParser.Engine = this; //aParser.RemoveErrorListeners(); //aParser.AddErrorListener(new ErrorListener()); RulezParser.RulezUnitContext aTree = aParser.rulezUnit(); // walk the parse tree -> get the result XPTree XPTGenerator aGenerator = new XPTGenerator(aParser); Antlr4.Runtime.Tree.ParseTreeWalker.Default.Walk(aGenerator, aTree); // result -> Generate and store from the XPTree return(Generate((IRule)aGenerator.XPTree)); } catch (Exception ex) { return(false); } }
public static string GetTextTillWhitespace(this Antlr4.Runtime.ICharStream stream) { var builder = new StringBuilder(); int la; for (int i = 1; (la = stream.La(i)) >= 0 && !char.IsWhiteSpace((char)la); i++) { builder.Append((char)la); } return(builder.ToString()); }
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span) { List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>(); if (_failedTimeout) { return(classificationSpans); } bool spanExtended = false; int extendMultilineSpanToLine = 0; SnapshotSpan extendedSpan = span; ITextSnapshot snapshot = span.Snapshot; ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState); using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250))) { Span requestedSpan = span; TState startState = AdjustParseSpan(classifierState, ref span); ICharStream input = CreateInputStream(span); ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState); lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer)); IToken previousToken = null; bool previousTokenEndsLine = false; /* this is held outside the loop because only tokens which end at the end of a line * impact its value. */ bool lineStateChanged = false; while (true) { IToken token = lexer.NextToken(); // The latter is true for EOF token with span.End at the end of the document bool inBounds = token.StartIndex < span.End.Position || token.StopIndex < span.End.Position; int startLineCurrent; if (token.Type == IntStreamConstants.Eof) { startLineCurrent = span.Snapshot.LineCount - 1; } else { startLineCurrent = token.Line - 1; } // endLinePrevious is the line number the previous token ended on int endLinePrevious; if (previousToken != null) { Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF"); endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex); } else { endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1; } if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) { int firstMultilineLine = endLinePrevious; if (previousToken == null || previousTokenEndsLine) { firstMultilineLine++; } for (int i = firstMultilineLine; i < startLineCurrent; i++) { if (!classifierState._lineStates[i].MultilineToken || lineStateChanged) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } if (IsMultilineToken(span.Snapshot, lexer, token)) { int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex); int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); for (int i = startLine; i < stopLine; i++) { if (!classifierState._lineStates[i].MultilineToken) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token); if (tokenEndsLine) { TState stateAtEndOfLine = lexer.GetCurrentState(); int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); lineStateChanged = classifierState._lineStates[line].MultilineToken || !_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine); // even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated. SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine)); if (lineStateChanged) { if (line < span.Snapshot.LineCount - 1) { /* update the span's end position or the line state change won't be reflected * in the editor */ int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } } } if (token.Type == IntStreamConstants.Eof) { break; } if (token.StartIndex >= span.End.Position) { break; } previousToken = token; previousTokenEndsLine = tokenEndsLine; if (token.StopIndex < requestedSpan.Start) { continue; } var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot); if (tokenClassificationSpans != null) { classificationSpans.AddRange(tokenClassificationSpans); } if (!inBounds) { break; } } } if (extendMultilineSpanToLine > 0) { int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } if (spanExtended) { /* Subtract 1 from each of these because the spans include the line break on their last * line, forcing it to appear as the first position on the following line. */ int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End); int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1; // when considering the last line of a document, span and extendedSpan may end on the same line ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine)); } return(classificationSpans); }
protected abstract ITokenSourceWithState <TState> CreateLexer(ICharStream input, int startLine, TState startState);
public static int LA(this Antlr4.Runtime.ICharStream cs, int la) { return(cs.La(la)); }
public JavaUnicodeStreamV4(ICharStream source) { Contract.Requires <ArgumentNullException>(source != null, "source"); _source = source; }
public JavaUnicodeStreamV4(ICharStream source) { Contract.Requires<ArgumentNullException>(source != null, "source"); _source = source; }
public virtual LexerInterpreter CreateLexerInterpreter(ICharStream input) { if (this.IsParser()) { throw new InvalidOperationException("A lexer interpreter can only be created for a lexer or combined grammar."); } if (this.IsCombined()) { return implicitLexer.CreateLexerInterpreter(input); } char[] serializedAtn = ATNSerializer.GetSerializedAsChars(atn, GetRuleNames()); ATN deserialized = new ATNDeserializer().Deserialize(serializedAtn); return new LexerInterpreter(fileName, GetVocabulary(), GetRuleNames(), ((LexerGrammar)this).modes.Keys, deserialized, input); }