public TokenizationOptions(
TokenizerKind tokenizerKind,
bool splitOnPunctuation = true,
char[] additionalSplitCharacters = null,
bool caseInsensitive = true,
bool accentInsensitive = true)
{
this.TokenizerKind = tokenizerKind;
this.SplitOnPunctuation = splitOnPunctuation;
this.AdditionalSplitCharacters = additionalSplitCharacters;
this.CaseInsensitive = caseInsensitive;
this.AccentInsensitive = accentInsensitive;
}
private static ITokenizer CreateTokenizer(TokenizerKind tokenizerKind)
{
switch (tokenizerKind)
{
case TokenizerKind.Default:
return(new BasicTokenizer());
case TokenizerKind.XmlContent:
return(new XmlTokenizer());
default:
throw new LiftiException(ExceptionMessages.UnsupportedTokenizerKind, tokenizerKind);
}
}
public TokenizationOptions(TokenizerKind tokenizerKind)
{
this.TokenizerKind = tokenizerKind;
}
