public SynSetGraph(Token[] tokens)
{
Wnlib.WNCommon.path = "C:\\Program Files\\WordNet\\2.1\\dict\\";
m_tokens = tokens;
m_htMap = new Hashtable(1024);
}
public Token[] Tokenize(string input)
{
ArrayList tokens = new ArrayList();
//input = NormalizeCasing(input);
input = m_trunc.Replace(input, string.Empty);
string[] rawTokens = m_splitter.Split(input);
for (int i = 0; i < rawTokens.Length; i++)
{
// Break up brill tags
string []twoKons = rawTokens[i].Split('/');
if (twoKons.Length == 2) // TODO: Fix this so that / is allowed in text (?)
{
//m_trunc.Replace(twoKons[0], "");
if (!StopWordsHandler.Instance.IsStopWord(twoKons[0]))
{
Token t = new Token(twoKons[0], twoKons[1]);
tokens.Add(t);
}
}
}
return (Token[]) tokens.ToArray(typeof(Token));
}
