/// <summary>
/// Create a new lexer
/// </summary>
/// <param name="code">the code to be parsed</param>
/// <param name="keywords">a list of language keywords</param>
/// <param name="tokens">a list of regexes and token types to compare the code to</param>
/// <param name="comment_sequence">the language sequence for ignorable comments</param>
/// <param name="include_sequence">The language sequence for includation of other files</param>
public RegexLexer(string code, List <string> keywords, Dictionary <string, string> tokens,
string comment_sequence, string include_sequence)
{
this.manager = new RegexCodeTokenizer(code: code);
this.keywords = keywords;
this.allowed_tokens = new SortedDictionary <string, string>(tokens, new LengthComparer());
this.comment_sequence = comment_sequence;
this.include_sequence = include_sequence;
}
/// <summary>
/// A lexer that uses the default tokens to parse
/// </summary>
/// <param name="code">the code to be parsed</param>
public RegexLexer(string code)
{
manager = new RegexCodeTokenizer(code: code);
}
