public Tokenizer(IDictionary dict) { dictionary = dict; string regexp = dict.GetWordBoundaryRegex(); if (!String.IsNullOrEmpty(regexp)) { rg = new Regex(@"(\W*)("+regexp+@")(\W*)", RegexOptions.Compiled | RegexOptions.IgnoreCase); } else { rg = new Regex(@"(\W*)([a-z-]*'*[a-z]+)(\W*)", RegexOptions.Compiled | RegexOptions.IgnoreCase); } }