/// <summary>
/// Returns the CodeFragment at the given position if such a fragment exists and null otherwise.
/// If the given position is equal to the end of the fragment, that fragment is returned if includeEnd is set to true.
/// If a fragment is determined for the given position, returns the corresponding token index as out parameter.
/// Note that token indices are no longer valid as soon as the file is modified (possibly e.g. by queued background processing).
/// Any query or attempt operation on the returned token index may result in an exception once it lost its validity.
/// Returns null if the given file or the specified position is null,
/// or if the specified position is not within the current Content range.
/// </summary>
public static CodeFragment?TryGetFragmentAt(
this FileContentManager file,
Position?pos,
out CodeFragment.TokenIndex?tIndex,
bool includeEnd = false)
{
tIndex = null;
if (file == null || pos == null || !file.ContainsPosition(pos))
{
return(null);
}
var start = pos.Line;
var previous = file.GetTokenizedLine(start).Where(token => token.Range.Start.Column <= pos.Column).ToImmutableArray();
while (!previous.Any() && --start >= 0)
{
previous = file.GetTokenizedLine(start);
}
if (!previous.Any())
{
return(null);
}
var lastPreceding = previous.Last().WithLineNumOffset(start);
var overlaps = includeEnd
? pos <= lastPreceding.Range.End
: pos < lastPreceding.Range.End;
tIndex = overlaps ? new CodeFragment.TokenIndex(file, start, previous.Length - 1) : null;
return(overlaps ? lastPreceding : null);
}
/// <summary>
/// Returns true if the given file contains any tokens overlapping with the given fragment.
/// The range of the tokens in the file is assumed to be relative to their start line (the index at which they are listed),
/// whereas the range of the given fragment is assumed to be the absolute range.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="range"/> is not a valid range within <paramref name="file"/>.</exception>
internal static bool ContainsTokensOverlappingWith(this FileContentManager file, Range range)
{
if (!file.ContainsRange(range))
{
throw new ArgumentOutOfRangeException(nameof(range));
}
var(start, end) = (range.Start.Line, range.End.Line);
if (start != end && file.GetTokenizedLines(start + 1, end - start - 1).SelectMany(x => x).Any())
{
return(true);
}
var inRange = file.GetTokenizedLine(start).Where(TokensAfter(Position.Create(0, range.Start.Column))); // checking tokens overlapping with range.Start below
inRange = start == end
? inRange.Where(TokensStartingBefore(Position.Create(0, range.End.Column)))
: inRange.Concat(file.GetTokenizedLine(end).Where(TokensStartingBefore(Position.Create(0, range.End.Column))));
if (inRange.Any())
{
QsCompilerError.Raise($"{range.DiagnosticString()} overlaps for start = {start}, end = {end}, \n\n" +
$"{string.Join("\n", file.GetTokenizedLine(start).Select(x => $"{x.Range.DiagnosticString()}"))},\n\n " +
$"{string.Join("\n", file.GetTokenizedLine(end).Select(x => $"{x.Range.DiagnosticString()}"))},");
return(true);
}
var overlapsWithStart = file.TryGetFragmentAt(range.Start, out _);
return(overlapsWithStart != null);
}
/// <summary>
/// Returns all code fragments in the specified file that overlap with the given range.
/// Returns an empty sequence if any of the given arguments is null.
/// </summary>
private static IEnumerable <CodeFragment> FragmentsOverlappingWithRange(this FileContentManager file, LSP.Range range)
{
if (file == null || range?.Start == null || range.End == null)
{
return(Enumerable.Empty <CodeFragment>());
}
var(start, end) = (range.Start.Line, range.End.Line);
var fragAtStart = file.TryGetFragmentAt(range.Start, out var _, includeEnd: true);
var inRange = file.GetTokenizedLine(start).Select(t => t.WithUpdatedLineNumber(start)).Where(ContextBuilder.TokensAfter(range.Start)); // does not include fragAtStart
inRange = start == end
? inRange.Where(ContextBuilder.TokensStartingBefore(range.End))
: inRange.Concat(file.GetTokenizedLines(start + 1, end - start - 1).SelectMany((x, i) => x.Select(t => t.WithUpdatedLineNumber(start + 1 + i))))
.Concat(file.GetTokenizedLine(end).Select(t => t.WithUpdatedLineNumber(end)).Where(ContextBuilder.TokensStartingBefore(range.End)));
var fragments = ImmutableArray.CreateBuilder <CodeFragment>();
if (fragAtStart != null)
{
fragments.Add(fragAtStart);
}
fragments.AddRange(inRange);
return(fragments.ToImmutableArray());
}
/// <summary>
/// Returns the token index at, or the closest token index before, the given position. Returns null if there is
/// no token at or before the given position.
/// </summary>
/// <exception cref="ArgumentNullException">Thrown when any argument is null.</exception>
/// <exception cref="ArgumentException">Thrown when the position is invalid.</exception>
private static CodeFragment.TokenIndex GetTokenAtOrBefore(FileContentManager file, Position position)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
if (!Utils.IsValidPosition(position))
{
throw new ArgumentException(nameof(position));
}
var line = position.Line;
var tokens = file.GetTokenizedLine(line);
// If the current line is empty, find the last non-empty line before it.
while (tokens.IsEmpty && line > 0)
{
tokens = file.GetTokenizedLine(--line);
}
var index = tokens.TakeWhile(ContextBuilder.TokensUpTo(position)).Count() - 1;
if (index == -1)
{
return(null);
}
return(new CodeFragment.TokenIndex(file, line, index));
}
/// <summary>
/// Returns the CodeFragment at the given position if such a fragment exists and null otherwise.
/// If the given position is equal to the end of the fragment, that fragment is returned if includeEnd is set to true.
/// If a fragment is determined for the given position, returns the corresponding token index as out parameter.
/// Note that token indices are no longer valid as soon as the file is modified (possibly e.g. by queued background processing).
/// Any query or attempt operation on the returned token index may result in an exception once it lost its validity.
/// Returns null if the given file or the specified position is null,
/// or if the specified position is not within the current Content range.
/// </summary>
public static CodeFragment TryGetFragmentAt(this FileContentManager file, Position pos,
out CodeFragment.TokenIndex tIndex, bool includeEnd = false)
{
tIndex = null;
if (file == null || pos == null || !Utils.IsValidPosition(pos, file))
{
return(null);
}
var start = pos.Line;
var previous = file.GetTokenizedLine(start).Where(token => token.GetRange().Start.Character <= pos.Character).ToImmutableArray();
while (!previous.Any() && --start >= 0)
{
previous = file.GetTokenizedLine(start);
}
if (!previous.Any())
{
return(null);
}
var lastPreceding = previous.Last().WithUpdatedLineNumber(start);
var overlaps = includeEnd
? pos.IsSmallerThanOrEqualTo(lastPreceding.GetRange().End)
: pos.IsSmallerThan(lastPreceding.GetRange().End);
tIndex = overlaps ? new CodeFragment.TokenIndex(file, start, previous.Length - 1) : null;
return(overlaps ? lastPreceding : null);
}
/// <summary>
/// Returns the CodeFragment at the given position if such a fragment exists and null otherwise.
/// If the given position is equal to the end of the fragment, that fragment is returned if includeEnd is set to true.
/// Returns null if the given file or the specified position is null,
/// or if the specified position is not within the current Content range.
/// </summary>
public static CodeFragment TryGetFragmentAt(this FileContentManager file, Position pos, bool includeEnd = false)
{
if (file == null || pos == null || !Utils.IsValidPosition(pos, file))
{
return(null);
}
var start = pos.Line;
var previous = file.GetTokenizedLine(start).Where(token => token.GetRange().Start.IsSmallerThanOrEqualTo(new Position(0, pos.Character)));
while (!previous.Any() && --start >= 0)
{
previous = file.GetTokenizedLine(start);
}
if (!previous.Any())
{
return(null);
}
var lastPreceding = previous.Last().WithUpdatedLineNumber(start);
var overlaps = includeEnd
? pos.IsSmallerThanOrEqualTo(lastPreceding.GetRange().End)
: pos.IsSmallerThan(lastPreceding.GetRange().End);
return(overlaps ? lastPreceding : null);
}
/// <summary>
/// Returns the TokenIndex for the last token in the given file, or null if no such token exists.
/// </summary>
internal static CodeFragment.TokenIndex?LastToken(this FileContentManager file)
{
var lastNonEmpty = file.NrLines();
while (lastNonEmpty-- > 0 && file.GetTokenizedLine(lastNonEmpty).Length == 0)
{
}
return(lastNonEmpty < 0
? null
: new CodeFragment.TokenIndex(file, lastNonEmpty, file.GetTokenizedLine(lastNonEmpty).Length - 1));
}
/// <summary>
/// Returns the TokenIndex for the last token in the given file, or null if no such token exists.
/// Throws an ArgumentNullException if file is null.
/// </summary>
internal static CodeFragment.TokenIndex LastToken(this FileContentManager file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
var lastNonEmpty = file.NrLines();
while (lastNonEmpty-- > 0 && file.GetTokenizedLine(lastNonEmpty).Length == 0)
{
}
return(lastNonEmpty < 0
? null
: new CodeFragment.TokenIndex(file, lastNonEmpty, file.GetTokenizedLine(lastNonEmpty).Length - 1));
}
/// <summary>
/// Returns the TokenIndex for the first token in the given file, or null if no such token exists.
/// </summary>
internal static CodeFragment.TokenIndex?FirstToken(this FileContentManager file)
{
var lineNr = 0;
while (file.GetTokenizedLine(lineNr).Length == 0 && ++lineNr < file.NrLines())
{
}
return(lineNr == file.NrLines()
? null
: new CodeFragment.TokenIndex(file, lineNr, 0));
}
/// <summary>
/// Given an SortedSet of changed lines, verifies the context for each token on one of these lines,
/// and adds the computed diagnostics to the ones returned as out parameter.
/// Marks the token indices which are to be excluded from compilation due to context errors.
/// Returns the line numbers for which the context diagnostics have been recomputed.
/// </summary>
private static HashSet <int> VerifyContext(this FileContentManager file, SortedSet <int> changedLines, out List <Diagnostic> diagnostics)
{
IEnumerable <CodeFragment.TokenIndex> TokenIndices(int lineNr) =>
Enumerable.Range(0, file.GetTokenizedLine(lineNr).Count()).Select(index => new CodeFragment.TokenIndex(file, lineNr, index));
var tokensToVerify = changedLines.SelectMany(TokenIndices);
var verifiedLines = new HashSet <int>();
List <Diagnostic> Verify(CodeFragment.TokenIndex tokenIndex)
{
var messages = new List <Diagnostic>();
var fragment = tokenIndex.GetFragment();
var context = tokenIndex.GetContext();
var(include, verifications) = Context.VerifySyntaxTokenContext(context);
foreach (var msg in verifications)
{
messages.Add(Diagnostics.Generate(file.FileName, msg, fragment.Range.Start));
}
if (include)
{
tokenIndex.MarkAsIncluded();
}
else
{
tokenIndex.MarkAsExcluded();
}
verifiedLines.Add(tokenIndex.Line);
// if a token is newly included in or excluded from the compilation,
// then this may impact context information for all following tokens
var changedStatus = include ^ fragment.IncludeInCompilation;
var next = tokenIndex.NextOnScope();
if (changedStatus && next != null && !changedLines.Contains(next.Line))
{
// NOTE: since we invalidate context diagnostics on a per-line basis, we need to verify the whole line!
var tokens = TokenIndices(next.Line);
foreach (var token in tokens)
{
messages.AddRange(Verify(token));
}
}
return(messages);
}
diagnostics = tokensToVerify.SelectMany(tIndex => Verify(tIndex)).ToList();
return(verifiedLines);
}
/// <summary>
/// Verifies the given line number and index *only* against the Tokens listed in file (and not against the content)
/// and initializes an instance of TokenIndex.
/// Throws an ArgumentNullException if file is null.
/// Throws an ArgumentOutOfRangeException if line or index are negative,
/// or line is larger than or equal to the number of Tokens lists in file,
/// or index is larger than or equal to the number of Tokens on the given line.
/// </summary>
internal TokenIndex(FileContentManager file, int line, int index)
{
this.File = file ?? throw new ArgumentNullException(nameof(file));
if (line < 0 || line >= file.NrTokenizedLines())
{
throw new ArgumentOutOfRangeException(nameof(line));
}
if (index < 0 || index >= file.GetTokenizedLine(line).Length)
{
throw new ArgumentOutOfRangeException(nameof(index));
}
this.Line = line;
this.Index = index;
}
/// <summary>
/// Returns the TokenIndex for the first token in the given file, or null if no such token exists.
/// Throws an ArgumentNullException if file is null.
/// </summary>
internal static CodeFragment.TokenIndex FirstToken(this FileContentManager file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
var lineNr = 0;
while (file.GetTokenizedLine(lineNr).Length == 0 && ++lineNr < file.NrLines())
{
}
return(lineNr == file.NrLines()
? null
: new CodeFragment.TokenIndex(file, lineNr, 0));
}
/// <summary>
/// Verifies the given line number and index *only* against the Tokens listed in file (and not against the
/// content) and initializes an instance of TokenIndex.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException">The line or index are negative.</exception>
/// <exception cref="FileContentException">
/// The line is outside the bounds of the file, or the index is outside the bounds of the line.
/// </exception>
internal TokenIndex(FileContentManager file, int line, int index)
{
if (line < 0)
{
throw new ArgumentOutOfRangeException(nameof(line));
}
if (index < 0)
{
throw new ArgumentOutOfRangeException(nameof(index));
}
if (line >= file.NrTokenizedLines())
{
throw new FileContentException("Line exceeds the bounds of the file.");
}
if (index >= file.GetTokenizedLine(line).Length)
{
throw new FileContentException("Token exceeds the bounds of the line.");
}
this.file = file;
this.Line = line;
this.Index = index;
}
