public TokenPositionAnalysis(Token t, int wsPrediction, string wsAnalysis, int alignPrediction, string alignAnalysis) { this.t = t; this.wsPrediction = wsPrediction; this.wsAnalysis = wsAnalysis; this.alignPrediction = alignPrediction; this.alignAnalysis = alignAnalysis; }
public Source( Antlr4.Runtime.IToken token ) { source = ""; StartLine = token.Line; StartColumn = token.StartIndex; EndLine = token.Line; EndColumn = token.StopIndex; }
/// <summary> /// If the source is user-input /// </summary> public Source( Antlr4.Runtime.IToken start, Antlr4.Runtime.IToken end ) { source = ""; StartLine = start.Line; StartColumn = start.Column; EndLine = end.Line; EndColumn = end.Column; }
// Extension to get safe line/column numbers internal static LinePosition GetLinePosition(this Antlr4.Runtime.IToken token, bool start) { int iLine = token.Line > 1 ? token.Line - 1 : 0; // note Antlr Line = 1 based int iCol = token.Column > 0 ? token.Column : 0; // note Antlr Column = 0 based if (!start) { iCol += token.Text.Length; } return(new LinePosition(iLine, iCol)); }
/// <summary> /// If the source is a file /// </summary> public Source( string filePath, Antlr4.Runtime.IToken start, Antlr4.Runtime.IToken end ) { source = filePath; StartLine = start.Line; StartColumn = start.Column; EndLine = end.Line; EndColumn = end.Column; }
protected virtual IEnumerable <ITagSpan <TTag> > GetTagSpansForToken([NotNull] IToken token, [NotNull] ITextSnapshot snapshot) { Requires.NotNull(token, nameof(token)); Requires.NotNull(snapshot, nameof(snapshot)); TTag tag; if (TryTagToken(token, out tag)) { SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1); return(new ITagSpan <TTag>[] { new TagSpan <TTag>(span, tag) }); } return(Enumerable.Empty <ITagSpan <TTag> >()); }
protected virtual IEnumerable <ITagSpan <TTag> > GetTagSpansForToken(IToken token, ITextSnapshot snapshot) { Contract.Requires <ArgumentNullException>(token != null, "token"); Contract.Requires <ArgumentNullException>(snapshot != null, "snapshot"); Contract.Ensures(Contract.Result <IEnumerable <ITagSpan <TTag> > >() != null); TTag tag; if (TryTagToken(token, out tag)) { SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1); return(new ITagSpan <TTag>[] { new TagSpan <TTag>(span, tag) }); } return(Enumerable.Empty <ITagSpan <TTag> >()); }
public TypeReference ParseTypeString( string text, bool skipPrimitives = false, bool reportErrors = false, Antlr4.Runtime.IToken token = null) { int iBrackets = text.IndexOf('['); var identifier = (iBrackets >= 0) ? text.Substring(0, iBrackets) : text; if (!skipPrimitives) { switch (identifier) { case "void": return(TypeReference.TypeVoid); case "int": return(TypeReference.TypeInt64); case "integer": return(TypeReference.TypeInt64); case "string": return(TypeReference.TypeString); case "decimal": return(TypeReference.TypeDouble); case "double": return(TypeReference.TypeDouble); case "bool": return(TypeReference.TypeBool); case "verdict": return(TypeReference.TypeVerdict); case "timespan": return(TypeReference.TypeTimeSpan); case "datetime": return(TypeReference.TypeDateTime); case "object": return(TypeReference.TypeObject); case "procedure": return(TypeReference.TypeProcedure); case "function": return(TypeReference.TypeFunction); default: break; } } return(this.ResolveQualifiedType(identifier, reportErrors, token)?.DataType); }
public static IEnumerable <Location> GetDefs(Document doc) { List <Location> result = new List <Location>(); ParserDetails ref_pd = ParserDetailsFactory.Create(doc); if (ref_pd.ParseTree == null) { LanguageServer.Module.Compile(); } foreach (KeyValuePair <TerminalNodeImpl, int> value in ref_pd.Defs) { TerminalNodeImpl key = value.Key; Antlr4.Runtime.IToken sym = key.Payload; result.Add( new Location() { Range = new Workspaces.Range(sym.StartIndex, sym.StopIndex), Uri = Workspaces.Workspace.Instance.FindDocument(sym.InputStream.SourceName) }); } return(result); }
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span) { List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>(); if (_failedTimeout) { return(classificationSpans); } bool spanExtended = false; int extendMultilineSpanToLine = 0; SnapshotSpan extendedSpan = span; ITextSnapshot snapshot = span.Snapshot; ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState); using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250))) { Span requestedSpan = span; TState startState = AdjustParseSpan(classifierState, ref span); ICharStream input = CreateInputStream(span); ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState); lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer)); IToken previousToken = null; bool previousTokenEndsLine = false; /* this is held outside the loop because only tokens which end at the end of a line * impact its value. */ bool lineStateChanged = false; while (true) { IToken token = lexer.NextToken(); // The latter is true for EOF token with span.End at the end of the document bool inBounds = token.StartIndex < span.End.Position || token.StopIndex < span.End.Position; int startLineCurrent; if (token.Type == IntStreamConstants.Eof) { startLineCurrent = span.Snapshot.LineCount - 1; } else { startLineCurrent = token.Line - 1; } // endLinePrevious is the line number the previous token ended on int endLinePrevious; if (previousToken != null) { Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF"); endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex); } else { endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1; } if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) { int firstMultilineLine = endLinePrevious; if (previousToken == null || previousTokenEndsLine) { firstMultilineLine++; } for (int i = firstMultilineLine; i < startLineCurrent; i++) { if (!classifierState._lineStates[i].MultilineToken || lineStateChanged) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } if (IsMultilineToken(span.Snapshot, lexer, token)) { int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex); int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); for (int i = startLine; i < stopLine; i++) { if (!classifierState._lineStates[i].MultilineToken) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token); if (tokenEndsLine) { TState stateAtEndOfLine = lexer.GetCurrentState(); int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); lineStateChanged = classifierState._lineStates[line].MultilineToken || !_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine); // even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated. SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine)); if (lineStateChanged) { if (line < span.Snapshot.LineCount - 1) { /* update the span's end position or the line state change won't be reflected * in the editor */ int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } } } if (token.Type == IntStreamConstants.Eof) { break; } if (token.StartIndex >= span.End.Position) { break; } previousToken = token; previousTokenEndsLine = tokenEndsLine; if (token.StopIndex < requestedSpan.Start) { continue; } var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot); if (tokenClassificationSpans != null) { classificationSpans.AddRange(tokenClassificationSpans); } if (!inBounds) { break; } } } if (extendMultilineSpanToLine > 0) { int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } if (spanExtended) { /* Subtract 1 from each of these because the spans include the line break on their last * line, forcing it to appear as the first position on the following line. */ int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End); int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1; // when considering the last line of a document, span and extendedSpan may end on the same line ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine)); } return(classificationSpans); }
private AST.SourceInfo GetSourceInfo(Antlr4.Runtime.IToken token) { return(new AST.SourceInfo(token.Line, token.Column)); }
protected virtual IEnumerable <ClassificationSpan> GetClassificationSpansForToken([NotNull] IToken token, [NotNull] ITextSnapshot snapshot) { Requires.NotNull(token, nameof(token)); Requires.NotNull(snapshot, nameof(snapshot)); var classification = ClassifyToken(token); if (classification != null) { SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1); return(new ClassificationSpan[] { new ClassificationSpan(span, classification) }); } return(Enumerable.Empty <ClassificationSpan>()); }
protected virtual IClassificationType ClassifyToken([NotNull] IToken token) { Requires.NotNull(token, nameof(token)); return(null); }
protected virtual bool IsMultilineToken(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token) { if (token.Type == IntStreamConstants.Eof) { return(false); } int startLine = snapshot.GetLineNumberFromPosition(token.StartIndex); int stopLine = snapshot.GetLineNumberFromPosition(token.StopIndex); return(startLine != stopLine); }
protected virtual bool TokenEndsAtEndOfLine(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token) { if (token.StopIndex + 1 >= snapshot.Length) { return(true); } char c = snapshot[token.StopIndex + 1]; return(c == '\r' || c == '\n'); }
public override void SyntaxError(Antlr4.Runtime.IRecognizer recognizer, Antlr4.Runtime.IToken offendingSymbol, int line, int charPositionInLine, string msg, Antlr4.Runtime.RecognitionException e) { m_reporter.GenerateError(msg, line, charPositionInLine); }
protected virtual bool TokensSkippedLines(ITextSnapshot snapshot, int endLinePrevious, IToken token) { int startLineCurrent = snapshot.GetLineNumberFromPosition(token.StartIndex); return(startLineCurrent > endLinePrevious + 1); }
protected virtual IEnumerable <ClassificationSpan> GetClassificationSpansForToken(IToken token, ITextSnapshot snapshot) { Contract.Requires <ArgumentNullException>(token != null, "token"); Contract.Requires <ArgumentNullException>(snapshot != null, "snapshot"); Contract.Ensures(Contract.Result <IEnumerable <ClassificationSpan> >() != null); var classification = ClassifyToken(token); if (classification != null) { SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1); return(new ClassificationSpan[] { new ClassificationSpan(span, classification) }); } return(Enumerable.Empty <ClassificationSpan>()); }
protected virtual bool TryTagToken([NotNull] IToken token, out TTag tag) { Requires.NotNull(token, nameof(token)); tag = default(TTag); return(false); }
public VariableAlreadyDefinedException(Antlr4.Runtime.IToken token) { // TODO: Complete member initialization this.token = token; }
public void FindPartitions() { var find_first_rule = new FindFirstRule(); ParseTreeWalker.Default.Walk(find_first_rule, pd_parser.ParseTree); var first_rule = find_first_rule.First; if (first_rule == null) { return; } var insertion = first_rule.SourceInterval.a; var insertion_tok = pd_parser.TokStream.Get(insertion); var insertion_ind = insertion_tok.StartIndex; string old_code = document.Code; for (int i = 0; i < rules.Count; ++i) { IParseTree rule = rules[i].rule; // Find range indices for rule including comments. Note, start index is inclusive; end // index is exclusive. We make the assumption // that the preceeding whitespace and comments are grouped with a rule all the way // from the end a previous non-whitespace or comment, such as options, headers, or rule. Interval token_interval = rule.SourceInterval; var end = token_interval.b; var end_tok = pd_parser.TokStream.Get(end); Antlr4.Runtime.IToken last = end_tok; var end_ind = old_code.Length <= last.StopIndex ? last.StopIndex : last.StopIndex + 1; bool on_end = false; for (int j = end_ind; j < old_code.Length; j++) { if (old_code[j] == '\r') { if (j + 1 < old_code.Length && old_code[j + 1] == '\n') { end_ind = j + 2; } else { end_ind = j + 1; } break; } end_ind = j; } var inter = pd_parser.TokStream.GetHiddenTokensToRight(end_tok.TokenIndex); var start = token_interval.a; var start_tok = pd_parser.TokStream.Get(start); var start_ind = start_tok.StartIndex; rules[i].start_index = start_ind; rules[i].end_index = end_ind; } for (int i = 0; i < rules.Count; ++i) { if (i > 0) { rules[i].start_index = rules[i - 1].end_index; } } bool bad = false; for (int i = 0; i < rules.Count; ++i) { for (int j = rules[i].start_index; j < rules[i].end_index; ++j) { if (old_code[j] == '\r') { if (j + 1 < rules[i].end_index && old_code[j + 1] == '\n') { ; } else { bad = true; } } } } }
public UndeclaredVariableException(Antlr4.Runtime.IToken varNameToken) { // TODO: Complete member initialization this.varNameToken = varNameToken; }
protected virtual bool TryTagToken(IToken token, out TTag tag) { Contract.Requires <ArgumentNullException>(token != null, "token"); tag = default(TTag); return(false); }
protected virtual IClassificationType ClassifyToken(IToken token) { Contract.Requires <ArgumentNullException>(token != null, "token"); return(null); }