public Tokenizer(IDictionary dict)
{
dictionary = dict;
string regexp = dict.GetWordBoundaryRegex();
if (!String.IsNullOrEmpty(regexp))
{
rg = new Regex(@"(\W*)("+regexp+@")(\W*)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
else
{
rg = new Regex(@"(\W*)([a-z-]*'*[a-z]+)(\W*)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
}