public static void TestTokenCallBacks()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CallbackTokens> >());
if (!res.IsError)
{
var lexer = res.Result as GenericLexer <CallbackTokens>;
CallBacksBuilder.BuildCallbacks <CallbackTokens>(lexer);
var tokens = lexer.Tokenize("aaa bbb").ToList();
;
foreach (var token in tokens)
{
Console.WriteLine($"{token.TokenID} - {token.Value}");
}
}
}
public void TestTokenCallbacks()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CallbackTokens> >());
Assert.False(res.IsError);
var lexer = res.Result as GenericLexer <CallbackTokens>;
CallBacksBuilder.BuildCallbacks <CallbackTokens>(lexer);
var tokens = lexer.Tokenize("aaa bbb").ToList();
Assert.Equal(3, tokens.Count);
Assert.Equal("AAA", tokens[0].Value);
Assert.Equal("BBB", tokens[1].Value);
Assert.Equal(CallbackTokens.SKIP, tokens[1].TokenID);
}
