public Tokenizer(TokenizerContext context, bool ignoreWhitespace) { string whitespaceDelimiter = ignoreWhitespace ? @"\s*" : @""; Context = context; Rule = new Regex($@"{whitespaceDelimiter}(?:{ string.Join(@"|", Context.Select(tokenResource => $@"({ string.Join(@"|", tokenResource.RegexPatterns.Select(tokenPattern => $@"(?:{tokenPattern})")) })")) }|(.)){whitespaceDelimiter}", RegexOptions.Compiled); }
public Tokenizer(TokenizerContext context) : this(context, true) { }