public static ImmutableDictionary <int, TokenSlot> CreateTokens(GdCommonTokenStream tokenstream, List <IErrorNode> errorNodes)
{
var rawTokens = tokenstream.AllTokens;
var tokens = ImmutableDictionary.CreateBuilder <int, TokenSlot>();
var skippedTokens = new List <IToken>(errorNodes.Select(n => n.Symbol));
var tokenBuilder = new TokenBuilder();
foreach (var rawToken in rawTokens)
{
var kind = GetTokenKind(rawToken);
var extent = TextExtentFactory.CreateExtent(rawToken);
var isSkipedTokenTrivia = skippedTokens.Contains(rawToken);
if (rawToken.Channel == GdTokens.TriviaChannel ||
rawToken.Channel == Lexer.Hidden ||
isSkipedTokenTrivia)
{
var trivia = TriviaSlot.Create(rawToken.Text, kind, isSkipedTokenTrivia);
tokenBuilder.AddTrivia(trivia);
}
else
{
tokenBuilder.AddToken(extent.Start, rawToken.Text, kind, out var completedTokenInfo);
if (completedTokenInfo.Token != null)
{
tokens.Add(completedTokenInfo.Start, completedTokenInfo.Token);
}
}
}
var lastTokenInfo = tokenBuilder.TryCompleteToken();
if (lastTokenInfo.Token != null)
{
tokens.Add(lastTokenInfo.Start, lastTokenInfo.Token);
}
return(tokens.ToImmutable());
}
static SyntaxTree ParseInternal(SourceText sourceText,
[CanBeNull] Func <GdGrammar, ParserRuleContext> treeCreator,
CancellationToken cancellationToken)
{
treeCreator ??= (p => p.guiDescription());
var diagnostics = ImmutableArray.CreateBuilder <Diagnostic>();
// Setup Lexer
var stream = sourceText.ToCharStream();
var lexer = new GdTokens(stream);
var lexerErrorListener = new GdLexerErrorListener(sourceText, diagnostics);
lexer.RemoveErrorListeners();
lexer.AddErrorListener(lexerErrorListener);
// Setup Parser
var cts = new GdCommonTokenStream(lexer);
var parser = new GdGrammar(cts);
var parserErrorListener = new GdParserErrorListener(sourceText, diagnostics);
parser.RemoveErrorListeners();
parser.AddErrorListener(parserErrorListener);
var tree = treeCreator(parser);
cancellationToken.ThrowIfCancellationRequested();
var errorNodes = new ErrorNodeVisitor();
errorNodes.Visit(tree);
cancellationToken.ThrowIfCancellationRequested();
var tokens = TokenFactory.CreateTokens(cts, errorNodes.ErrorNodes);
var visitor = new GdSyntaxSlotBuilder(tokens);
var slot = visitor.Visit(tree);
return(new SyntaxTree(sourceText, slot, diagnostics.ToImmutableArray()));
}
