protected SyntaxParser(
Lexer lexer,
Lua.LuaSyntaxNode oldTree,
IEnumerable <TextChangeRange> changes,
bool preLexIfNotIncremental = false,
CancellationToken cancellationToken = default(CancellationToken))
{
_lexer = lexer;
_cancellationToken = cancellationToken;
_currentNode = default(BlendedNode);
_isIncremental = oldTree != null;
if (IsIncremental)
{
_firstBlender = new Blender(lexer, oldTree, changes);
_blendedTokens = s_blendedNodesPool.Allocate();
}
else
{
_firstBlender = default(Blender);
_lexedTokens = new ArrayElement <SyntaxToken> [32];
}
// PreLex is not cancellable.
// If we may cancel why would we aggressively lex ahead?
// Cancellations in a constructor make disposing complicated
//
// So, if we have a real cancellation token, do not do prelexing.
if (preLexIfNotIncremental && !IsIncremental && !cancellationToken.CanBeCanceled)
{
PreLex();
}
}
public Blender(Lexer lexer, Lua.LuaSyntaxNode oldTree, IEnumerable <TextChangeRange> changes)
{
LorettaDebug.Assert(lexer != null);
_lexer = lexer;
_changes = ImmutableStack.Create <TextChangeRange>();
if (changes != null)
{
// TODO: Consider implementing NormalizedChangeCollection for TextSpan. the real
// reason why we are collapsing is because we want to extend change ranges and
// cannot allow them to overlap. This does not seem to be a big deal since multiple
// changes are infrequent and typically close to each other. However if we have
// NormalizedChangeCollection for TextSpan we can have both - we can extend ranges
// and not require collapsing them. NormalizedChangeCollection would also ensure
// that changes are always normalized.
// TODO: this is a temporary measure to prevent individual change spans from
// overlapping after they are widened to effective width (+1 token at the start).
// once we have normalized collection for TextSpan we will not need to collapse all
// the change spans.
var collapsed = TextChangeRange.Collapse(changes);
// extend the change to its affected range. This will make it easier
// to filter out affected nodes since we will be able simply check
// if node intersects with a change.
var affectedRange = ExtendToAffectedRange(oldTree, collapsed);
_changes = _changes.Push(affectedRange);
}
if (oldTree == null)
{
// start at lexer current position if no nodes specified
_oldTreeCursor = new Cursor();
_newPosition = lexer.TextWindow.Position;
}
else
{
_oldTreeCursor = Cursor.FromRoot(oldTree).MoveToFirstChild();
_newPosition = 0;
}
_changeDelta = 0;
}
private static TextChangeRange ExtendToAffectedRange(
Lua.LuaSyntaxNode oldTree,
TextChangeRange changeRange)
{
// we will increase affected range of the change by the number of lookahead tokens
// original code in Blender seem to imply the lookahead at the end of a node is 1 token
// max. TODO: 1 token lookahead seems a bit too optimistic. Increase if needed.
const int maxLookahead = 1;
// check if change is not after the end. TODO: there should be an assert somewhere about
// changes starting at least at the End of old tree
var lastCharIndex = oldTree.FullWidth - 1;
// Move the start of the change range so that it is contained within oldTree.
var start = Math.Max(Math.Min(changeRange.Span.Start, lastCharIndex), 0);
// the first iteration aligns us with the change start. subsequent iteration move us to
// the left by maxLookahead tokens. We only need to do this as long as we're not at the
// start of the tree. Also, the tokens we get back may be zero width. In that case we
// need to keep on looking backward.
for (var i = 0; start > 0 && i <= maxLookahead;)
{
var token = oldTree.FindToken(start, findInsideTrivia: false);
LorettaDebug.Assert(token.Kind() != SyntaxKind.None, "how could we not get a real token back?");
start = Math.Max(0, token.Position - 1);
// Only increment i if we got a non-zero width token. Otherwise, we want to just do
// this again having moved back one space.
if (token.FullWidth > 0)
{
i++;
}
}
var finalSpan = TextSpan.FromBounds(start, changeRange.Span.End);
var finalLength = changeRange.NewLength + (changeRange.Span.Start - start);
return(new TextChangeRange(finalSpan, finalLength));
}
private BlendedNode CreateBlendedNode(Lua.LuaSyntaxNode node, SyntaxToken token)
{
return(new BlendedNode(node, token,
new Blender(_lexer, _oldTreeCursor, _changes, _newPosition, _changeDelta)));
}
private static bool IsIncomplete(Lua.LuaSyntaxNode node) => // A node is incomplete if the last token in it is a missing token. Use the green // node to determine this as it's much faster than going through the red API. node.Green.GetLastTerminal().IsMissing;
public static Cursor FromRoot(Lua.LuaSyntaxNode node) => new(node, indexInParent : 0);