private List <WordInformation> GetWords()
{
var tokens = tokenizer.Tokenize(currentTokenizedSentence);
Words = TokensDictSearcher.ConvertTokensToWords(tokens, japDict);
if (unTokenizedWords.Count > 0)
{
Words.InsertRange(0, unTokenizedWords);
}
return(Words);
}
private void CreateOrMarkReduceWord(string wordSurface, string wordBaseform, List <JmdictEntity> matchWords)
{
if (IsCombineToNewWord())
{
CreateAndAddNewWord(wordSurface, wordBaseform, matchWords);
return;
}
//If not reduce and is still a valid longest word match then specified linkWordGroup
//to mark it up
if (TokensDictSearcher.IsValidLongestWord(wordSurface, matchWords.Count))
{
linkWordGroup++;
for (int i = startIndex; i <= endIndex; i++)
{
words[i].LinkWordGroup = linkWordGroup;
}
}
results.Add(words[startIndex]);
endIndex = startIndex;
}
