public List <NLPToken> Tokenize(string code)
{
var tokens = new List <NLPToken>();
Context = new NLPLexicalContext(code);
do
{
foreach (var lexer in Lexers)
{
var type = lexer.IdentifySymbol(Context);
if (type != NLPTokenTypes.Unknown)
{
tokens.Add(new NLPToken(type, Context.CurrentSymbol));
break;
}
}
}while(Context.Advance());
return(tokens);
}
public abstract NLPTokenTypes IdentifySymbol(NLPLexicalContext state);
