private void GetPossibleTagsRecursive(string lexeme, out string[] lemmas, out PersianPartOfSpeech[] tags, out double[] weights)
{
if (_baseTagger.IsKnown(lexeme))
{
_baseTagger.GetPossibleTags(lexeme, out lemmas, out tags, out weights);
return;
}
var patinfos = _lemmatizer.MatchForSuffix(lexeme);
if (patinfos.Length == 0)
{
tags = _config.DefaultTags;
weights = _config.DefaultWeights;
lemmas = new [] { lexeme };
return;
}
// Starting from the longest lemma (=baseWord)
for (int i = patinfos.Length - 1; i >= 0; i--)
{
var lemmaPat = patinfos[i];
if (_baseTagger.IsKnown(lemmaPat.BaseWord))
{
PersianPartOfSpeech[] baseTags;
double[] baseWeights;
_baseTagger.GetPossibleTags(lemmaPat.BaseWord, out lemmas, out baseTags, out baseWeights);
ApplyPersianDeclensionRules(baseTags, baseWeights, lemmaPat.Suffix, out tags, out weights);
return;
}
else
{
PersianPartOfSpeech[] baseTags;
double[] baseWeights;
GetPossibleTagsRecursive(lemmaPat.BaseWord, out lemmas, out baseTags, out baseWeights);
ApplyPersianDeclensionRules(baseTags, baseWeights, lemmaPat.Suffix, out tags, out weights);
if (tags[0] != PersianPartOfSpeech.Unknown)
{
return;
}
}
}
tags = _config.DefaultTags;
weights = _config.DefaultWeights;
lemmas = new [] { lexeme };
}
///<summary>
/// Extract POS tagged dictionary to a file
///</summary>
///<param name="fileName">File name</param>
///<returns>Tru on success</returns>
public bool ExtractPOSTaggedDictionary(string fileName)
{
PersianPOSTag pos;
int totalCount = m_wordList.Count;
int currentStep = 0;
double remainingPregress = (1.0 - ProgressPercent) / 0.95;
double denom = (2.0 / ((totalCount) * (totalCount + 1))) * remainingPregress;
foreach (KeyValuePair <string, int> pair in m_wordList)
{
ProgressPercent += ((double)currentStep * denom);
currentStep++;
bool curWordAdded = false;
string word = pair.Key;
if (!m_wordContainerExternal.Contain(word, out pos)) //external dictionary does not contains the word
{
ReversePatternMatcherPatternInfo[] suffixPatternArray = m_suffixRecognizer.MatchForSuffix(word);
if (suffixPatternArray.Length > 0)
{
foreach (ReversePatternMatcherPatternInfo suffixPattern in suffixPatternArray)
{
string stem = suffixPattern.BaseWord;
if (m_wordContainerExternal.Contain(stem, out pos)) //external dictionary contains the stem
{
curWordAdded = true;
AddWordToFinalList(stem, m_wordList[word], pos);
break;
}
else if (m_wordList.ContainsKey(stem))
{
curWordAdded = true;
AddToDictionary(stem, word);
break;
}
}
if (!curWordAdded)
{
AddToDictionary(word, word);
}
}
else
{
AddToDictionary(word, word);
}
}
else
{
//if external dictionary contains the word, add it to file
AddWordToFinalList(word, m_wordList[word], pos);
}
}
return(DumpFinalList(fileName));
}
public String lemmatize(String input)
{
var matcher = lemmatizer.MatchForSuffix(input);
if (matcher.Length > 0)
{
input = matcher[0].BaseWord;
}
return(input);
}
public static bool IsValidInDictionary(string word, List <string> dic, PersianSuffixLemmatizer suffixer, PruneType prouneType)
{
if (PruneType.NoPrune == prouneType)
{
return(false);
}
if (dic.Contains(word))
{
return(true);
}
else if (PruneType.Stem == prouneType)
{
ReversePatternMatcherPatternInfo[] inf = suffixer.MatchForSuffix(word);
foreach (ReversePatternMatcherPatternInfo info in inf)
{
if (dic.Contains(info.BaseWord))
{
return(true);
}
}
}
return(false);
}