public void AddTrivia(TriviaSlot trivia)
{
if (!HasToken)
{
_leadingTrivia.Add(trivia);
}
else
{
_trailingTrivia.Add(trivia);
}
}
public static ImmutableDictionary <int, TokenSlot> CreateTokens(GdCommonTokenStream tokenstream, List <IErrorNode> errorNodes)
{
var rawTokens = tokenstream.AllTokens;
var tokens = ImmutableDictionary.CreateBuilder <int, TokenSlot>();
var skippedTokens = new List <IToken>(errorNodes.Select(n => n.Symbol));
var tokenBuilder = new TokenBuilder();
foreach (var rawToken in rawTokens)
{
var kind = GetTokenKind(rawToken);
var extent = TextExtentFactory.CreateExtent(rawToken);
var isSkipedTokenTrivia = skippedTokens.Contains(rawToken);
if (rawToken.Channel == GdTokens.TriviaChannel ||
rawToken.Channel == Lexer.Hidden ||
isSkipedTokenTrivia)
{
var trivia = TriviaSlot.Create(rawToken.Text, kind, isSkipedTokenTrivia);
tokenBuilder.AddTrivia(trivia);
}
else
{
tokenBuilder.AddToken(extent.Start, rawToken.Text, kind, out var completedTokenInfo);
if (completedTokenInfo.Token != null)
{
tokens.Add(completedTokenInfo.Start, completedTokenInfo.Token);
}
}
}
var lastTokenInfo = tokenBuilder.TryCompleteToken();
if (lastTokenInfo.Token != null)
{
tokens.Add(lastTokenInfo.Start, lastTokenInfo.Token);
}
return(tokens.ToImmutable());
}