public LexerToken(string rawString, int startCharIdx, int endCharIdx)
{
rawWord = RawWord.new_FromString(rawString);
this.startCharIdx = startCharIdx;
this.endCharIdx = endCharIdx;
tokenType = TYPE.UNCLASSIFIED;
}
// Given a list of contiguous (traditional) words, either look it up
// and return it, or create it and return it.
public Word addWordFromStrings(string[] rawStrings)
{
// Convert to RawWord objects.
//List<RawWord[]> rawWordList = new List<RawWord[]>();
RawWord[] rawWords = new RawWord[rawStrings.Length];
int i = 0;
foreach (string s in rawStrings)
{
rawWords[i] = RawWord.new_FromString(s);
i++;
}
// rawWordList.Add(rawWords);
return(addWordFromRawWords(rawWords));
}
public Word word(string s)
{
return(wordTrie.getValue(new RawWord[] { RawWord.new_FromString(s) }));
}