public Recognizer(Core.Tokenization.TokenType t,
int priority,
string tokenClassName,
string recognizerName)
: this(t, priority, tokenClassName, recognizerName, false)
{
}
public DefaultThaiFallbackRecognizer(Core.Tokenization.TokenType t, int priority, System.Globalization.CultureInfo culture, Core.Resources.IResourceDataAccessor dataAccessor)
: base(t, priority, culture, dataAccessor, false)
{
// also used for Khmer now
// System.Diagnostics.Debug.Assert(culture.TwoLetterISOLanguageName.Equals("th", StringComparison.OrdinalIgnoreCase));
_IsFallbackRecognizer = true;
}
public DefaultFallbackRecognizer(Core.Tokenization.TokenType t, int priority, System.Globalization.CultureInfo culture, Core.Resources.IResourceDataAccessor dataAccessor, bool separateClitics)
: base(t, priority, null, "DefaultFallbackRecognizer")
{
// NOTE the token type is ignored in the implementation as the fallback recognizer will deliver multiple token types
_Culture = culture;
if (dataAccessor != null)
{
_Resources = new Sdl.LanguagePlatform.Lingua.Resources.LanguageResources(_Culture, dataAccessor);
}
_IsFallbackRecognizer = true;
// TODO test performance of building up the trie and matching instead fo using StartsWith() on
// the list of clitics
HashSet <string> leadingClitics = Core.CultureInfoExtensions.GetLeadingClitics(_Culture);
if (leadingClitics != null)
{
int p = 0;
_LeadingClitics = new Trie <char, int>();
foreach (string s in leadingClitics)
{
_LeadingClitics.Add(s.ToCharArray(), p++);
}
}
}
public DefaultChineseFallbackRecognizer(Core.Tokenization.TokenType t, int priority, System.Globalization.CultureInfo culture, Core.Resources.IResourceDataAccessor dataAccessor)
: base(t, priority, culture, dataAccessor, false)
{
int i = 0;
_IsFallbackRecognizer = true;
_DefaultPunctCharset = CharacterSetParser.Parse(_DEFAULT_PUNC_CS, ref i);
}
public WordlistRecognizer(Core.Tokenization.TokenType t, int priority, string tokenClassName, string recognizerName, Wordlist w)
: base(t, priority, tokenClassName, recognizerName)
{
if (w == null)
{
throw new ArgumentNullException("w");
}
_List = w;
}
public DefaultJapaneseFallbackRecognizer(Core.Tokenization.TokenType t, int priority, System.Globalization.CultureInfo culture, Core.Resources.IResourceDataAccessor dataAccessor)
: base(t, priority, culture, dataAccessor, false)
{
// TODO outsource the pattern into resource assembly, or make externally configurable?
_DefaultWordRegex = new System.Text.RegularExpressions.Regex(_DEFAULT_WORD_RX, System.Text.RegularExpressions.RegexOptions.ExplicitCapture);
int i = 0;
_IsFallbackRecognizer = true;
_DefaultPunctCharset = CharacterSetParser.Parse(_DEFAULT_PUNC_CS, ref i);
}
public Recognizer(Core.Tokenization.TokenType t,
int priority,
string tokenClassName,
string recognizerName,
bool autoSubstitutable)
{
_Type = t;
_Priority = priority;
_TokenClassName = tokenClassName;
_RecognizerName = recognizerName;
_AdditionalTerminators = null;
_IsFallbackRecognizer = false;
_OverrideFallbackRecognizer = false;
_AutoSubstitutable = autoSubstitutable;
}
public DefaultJAZHFallbackRecognizer(Core.Tokenization.TokenType t, int priority, System.Globalization.CultureInfo culture, Core.Resources.IResourceDataAccessor dataAccessor)
: base(t, priority, culture, dataAccessor, false)
{
_IsFallbackRecognizer = true;
}
public RegexRecognizer(Core.Tokenization.TokenType t, int priority, string tokenClassName, string recognizerName, bool autoSubstitutable)
: base(t, priority, tokenClassName, recognizerName, autoSubstitutable)
{
// TODO allow passing regex options?
_Patterns = new List <RegexRecognizerPattern>();
}
