protected LexerBase(SpecialTokenConfig specialTokens, IReadOnlyList <MatchableTokenKindSpec> tokenKindSpecs)
{
ArgumentCheck.NotNullOrEmptyOrWithNulls(tokenKindSpecs, nameof(tokenKindSpecs));
var tokenKinds = new List <MatchableTokenKind>(tokenKindSpecs.Count);
foreach (var tokenKindSpec in tokenKindSpecs)
{
var tokenKind = tokenKindSpec.TokenKind;
tokenKinds.Add(tokenKind);
if (tokenKindSpec.Skipped)
{
SkippableTokenKinds.Add(tokenKind);
}
}
TokenKinds = tokenKinds;
_unknown = specialTokens?.Unknown ?? TokenKind.Unknown;
}
public SpecialTokenConfig(SpecialTokenKind unknown)
{
Unknown = unknown;
}