public TokenPositionAnalysis(Token t, int wsPrediction, string wsAnalysis, int alignPrediction, string alignAnalysis)
{
this.t = t;
this.wsPrediction = wsPrediction;
this.wsAnalysis = wsAnalysis;
this.alignPrediction = alignPrediction;
this.alignAnalysis = alignAnalysis;
}
public Source(
Antlr4.Runtime.IToken token
)
{
source = "";
StartLine = token.Line;
StartColumn = token.StartIndex;
EndLine = token.Line;
EndColumn = token.StopIndex;
}
/// <summary>
/// If the source is user-input
/// </summary>
public Source(
Antlr4.Runtime.IToken start,
Antlr4.Runtime.IToken end
)
{
source = "";
StartLine = start.Line;
StartColumn = start.Column;
EndLine = end.Line;
EndColumn = end.Column;
}
// Extension to get safe line/column numbers
internal static LinePosition GetLinePosition(this Antlr4.Runtime.IToken token, bool start)
{
int iLine = token.Line > 1 ? token.Line - 1 : 0; // note Antlr Line = 1 based
int iCol = token.Column > 0 ? token.Column : 0; // note Antlr Column = 0 based
if (!start)
{
iCol += token.Text.Length;
}
return(new LinePosition(iLine, iCol));
}
/// <summary>
/// If the source is a file
/// </summary>
public Source(
string filePath,
Antlr4.Runtime.IToken start,
Antlr4.Runtime.IToken end
)
{
source = filePath;
StartLine = start.Line;
StartColumn = start.Column;
EndLine = end.Line;
EndColumn = end.Column;
}
protected virtual IEnumerable <ITagSpan <TTag> > GetTagSpansForToken([NotNull] IToken token, [NotNull] ITextSnapshot snapshot)
{
Requires.NotNull(token, nameof(token));
Requires.NotNull(snapshot, nameof(snapshot));
TTag tag;
if (TryTagToken(token, out tag))
{
SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1);
return(new ITagSpan <TTag>[] { new TagSpan <TTag>(span, tag) });
}
return(Enumerable.Empty <ITagSpan <TTag> >());
}
protected virtual IEnumerable <ITagSpan <TTag> > GetTagSpansForToken(IToken token, ITextSnapshot snapshot)
{
Contract.Requires <ArgumentNullException>(token != null, "token");
Contract.Requires <ArgumentNullException>(snapshot != null, "snapshot");
Contract.Ensures(Contract.Result <IEnumerable <ITagSpan <TTag> > >() != null);
TTag tag;
if (TryTagToken(token, out tag))
{
SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1);
return(new ITagSpan <TTag>[] { new TagSpan <TTag>(span, tag) });
}
return(Enumerable.Empty <ITagSpan <TTag> >());
}
public TypeReference ParseTypeString(
string text,
bool skipPrimitives = false,
bool reportErrors = false,
Antlr4.Runtime.IToken token = null)
{
int iBrackets = text.IndexOf('[');
var identifier = (iBrackets >= 0) ? text.Substring(0, iBrackets) : text;
if (!skipPrimitives)
{
switch (identifier)
{
case "void": return(TypeReference.TypeVoid);
case "int": return(TypeReference.TypeInt64);
case "integer": return(TypeReference.TypeInt64);
case "string": return(TypeReference.TypeString);
case "decimal": return(TypeReference.TypeDouble);
case "double": return(TypeReference.TypeDouble);
case "bool": return(TypeReference.TypeBool);
case "verdict": return(TypeReference.TypeVerdict);
case "timespan": return(TypeReference.TypeTimeSpan);
case "datetime": return(TypeReference.TypeDateTime);
case "object": return(TypeReference.TypeObject);
case "procedure": return(TypeReference.TypeProcedure);
case "function": return(TypeReference.TypeFunction);
default:
break;
}
}
return(this.ResolveQualifiedType(identifier, reportErrors, token)?.DataType);
}
public static IEnumerable <Location> GetDefs(Document doc)
{
List <Location> result = new List <Location>();
ParserDetails ref_pd = ParserDetailsFactory.Create(doc);
if (ref_pd.ParseTree == null)
{
LanguageServer.Module.Compile();
}
foreach (KeyValuePair <TerminalNodeImpl, int> value in ref_pd.Defs)
{
TerminalNodeImpl key = value.Key;
Antlr4.Runtime.IToken sym = key.Payload;
result.Add(
new Location()
{
Range = new Workspaces.Range(sym.StartIndex, sym.StopIndex),
Uri = Workspaces.Workspace.Instance.FindDocument(sym.InputStream.SourceName)
});
}
return(result);
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
if (_failedTimeout)
{
return(classificationSpans);
}
bool spanExtended = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
ITextSnapshot snapshot = span.Snapshot;
ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState);
using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250)))
{
Span requestedSpan = span;
TState startState = AdjustParseSpan(classifierState, ref span);
ICharStream input = CreateInputStream(span);
ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState);
lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer));
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
while (true)
{
IToken token = lexer.NextToken();
// The latter is true for EOF token with span.End at the end of the document
bool inBounds = token.StartIndex < span.End.Position ||
token.StopIndex < span.End.Position;
int startLineCurrent;
if (token.Type == IntStreamConstants.Eof)
{
startLineCurrent = span.Snapshot.LineCount - 1;
}
else
{
startLineCurrent = token.Line - 1;
}
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF");
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine))
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!classifierState._lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
for (int i = startLine; i < stopLine; i++)
{
if (!classifierState._lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (tokenEndsLine)
{
TState stateAtEndOfLine = lexer.GetCurrentState();
int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
lineStateChanged =
classifierState._lineStates[line].MultilineToken ||
!_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine));
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.Type == IntStreamConstants.Eof)
{
break;
}
if (token.StartIndex >= span.End.Position)
{
break;
}
previousToken = token;
previousTokenEndsLine = tokenEndsLine;
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
// when considering the last line of a document, span and extendedSpan may end on the same line
ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine));
}
return(classificationSpans);
}
private AST.SourceInfo GetSourceInfo(Antlr4.Runtime.IToken token)
{
return(new AST.SourceInfo(token.Line, token.Column));
}
protected virtual IEnumerable <ClassificationSpan> GetClassificationSpansForToken([NotNull] IToken token, [NotNull] ITextSnapshot snapshot)
{
Requires.NotNull(token, nameof(token));
Requires.NotNull(snapshot, nameof(snapshot));
var classification = ClassifyToken(token);
if (classification != null)
{
SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1);
return(new ClassificationSpan[] { new ClassificationSpan(span, classification) });
}
return(Enumerable.Empty <ClassificationSpan>());
}
protected virtual IClassificationType ClassifyToken([NotNull] IToken token)
{
Requires.NotNull(token, nameof(token));
return(null);
}
protected virtual bool IsMultilineToken(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
if (token.Type == IntStreamConstants.Eof)
{
return(false);
}
int startLine = snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = snapshot.GetLineNumberFromPosition(token.StopIndex);
return(startLine != stopLine);
}
protected virtual bool TokenEndsAtEndOfLine(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
if (token.StopIndex + 1 >= snapshot.Length)
{
return(true);
}
char c = snapshot[token.StopIndex + 1];
return(c == '\r' || c == '\n');
}
public override void SyntaxError(Antlr4.Runtime.IRecognizer recognizer, Antlr4.Runtime.IToken offendingSymbol, int line, int charPositionInLine, string msg, Antlr4.Runtime.RecognitionException e)
{
m_reporter.GenerateError(msg, line, charPositionInLine);
}
protected virtual bool TokensSkippedLines(ITextSnapshot snapshot, int endLinePrevious, IToken token)
{
int startLineCurrent = snapshot.GetLineNumberFromPosition(token.StartIndex);
return(startLineCurrent > endLinePrevious + 1);
}
protected virtual IEnumerable <ClassificationSpan> GetClassificationSpansForToken(IToken token, ITextSnapshot snapshot)
{
Contract.Requires <ArgumentNullException>(token != null, "token");
Contract.Requires <ArgumentNullException>(snapshot != null, "snapshot");
Contract.Ensures(Contract.Result <IEnumerable <ClassificationSpan> >() != null);
var classification = ClassifyToken(token);
if (classification != null)
{
SnapshotSpan span = new SnapshotSpan(snapshot, token.StartIndex, token.StopIndex - token.StartIndex + 1);
return(new ClassificationSpan[] { new ClassificationSpan(span, classification) });
}
return(Enumerable.Empty <ClassificationSpan>());
}
protected virtual bool TryTagToken([NotNull] IToken token, out TTag tag)
{
Requires.NotNull(token, nameof(token));
tag = default(TTag);
return(false);
}
public VariableAlreadyDefinedException(Antlr4.Runtime.IToken token)
{
// TODO: Complete member initialization
this.token = token;
}
public void FindPartitions()
{
var find_first_rule = new FindFirstRule();
ParseTreeWalker.Default.Walk(find_first_rule, pd_parser.ParseTree);
var first_rule = find_first_rule.First;
if (first_rule == null)
{
return;
}
var insertion = first_rule.SourceInterval.a;
var insertion_tok = pd_parser.TokStream.Get(insertion);
var insertion_ind = insertion_tok.StartIndex;
string old_code = document.Code;
for (int i = 0; i < rules.Count; ++i)
{
IParseTree rule = rules[i].rule;
// Find range indices for rule including comments. Note, start index is inclusive; end
// index is exclusive. We make the assumption
// that the preceeding whitespace and comments are grouped with a rule all the way
// from the end a previous non-whitespace or comment, such as options, headers, or rule.
Interval token_interval = rule.SourceInterval;
var end = token_interval.b;
var end_tok = pd_parser.TokStream.Get(end);
Antlr4.Runtime.IToken last = end_tok;
var end_ind = old_code.Length <= last.StopIndex ? last.StopIndex : last.StopIndex + 1;
bool on_end = false;
for (int j = end_ind; j < old_code.Length; j++)
{
if (old_code[j] == '\r')
{
if (j + 1 < old_code.Length && old_code[j + 1] == '\n')
{
end_ind = j + 2;
}
else
{
end_ind = j + 1;
}
break;
}
end_ind = j;
}
var inter = pd_parser.TokStream.GetHiddenTokensToRight(end_tok.TokenIndex);
var start = token_interval.a;
var start_tok = pd_parser.TokStream.Get(start);
var start_ind = start_tok.StartIndex;
rules[i].start_index = start_ind;
rules[i].end_index = end_ind;
}
for (int i = 0; i < rules.Count; ++i)
{
if (i > 0)
{
rules[i].start_index = rules[i - 1].end_index;
}
}
bool bad = false;
for (int i = 0; i < rules.Count; ++i)
{
for (int j = rules[i].start_index; j < rules[i].end_index; ++j)
{
if (old_code[j] == '\r')
{
if (j + 1 < rules[i].end_index && old_code[j + 1] == '\n')
{
;
}
else
{
bad = true;
}
}
}
}
}
public UndeclaredVariableException(Antlr4.Runtime.IToken varNameToken)
{
// TODO: Complete member initialization
this.varNameToken = varNameToken;
}
protected virtual bool TryTagToken(IToken token, out TTag tag)
{
Contract.Requires <ArgumentNullException>(token != null, "token");
tag = default(TTag);
return(false);
}
protected virtual IClassificationType ClassifyToken(IToken token)
{
Contract.Requires <ArgumentNullException>(token != null, "token");
return(null);
}
