public Tokenizer(TokenizerContext context, bool ignoreWhitespace)
{
string whitespaceDelimiter = ignoreWhitespace ? @"\s*" : @"";
Context = context;
Rule = new Regex($@"{whitespaceDelimiter}(?:{
string.Join(@"|", Context.Select(tokenResource => $@"({
string.Join(@"|", tokenResource.RegexPatterns.Select(tokenPattern => $@"(?:{tokenPattern})"))
})"))
}|(.)){whitespaceDelimiter}", RegexOptions.Compiled);
}
public Tokenizer(TokenizerContext context)
: this(context, true)
{
}
