public TokenObj(TokenObj inTokenObj)
{
index_ = inTokenObj.GetIndex(); // index of this token in the TextObj
pos_ = inTokenObj.GetPos(); // position
tag_ = inTokenObj.GetTag();
orgTokenStr_ = inTokenObj.GetOrgTokenStr(); // the org str, never change
tokenStr_ = inTokenObj.GetTokenStr(); // the current str
// history of processes
procHist_ = new List <string>(inTokenObj.GetProcHist());
}
public static List <TokenObj> FlatTokenToArrayList(TokenObj inTokenObj)
{
string[] tokenArray = inTokenObj.GetTokenStr().Split(patternStrSpace_, true);
List <TokenObj> tokenArrayList = new List <TokenObj>();
foreach (string tokenStr in tokenArray)
{
TokenObj tokenObj = new TokenObj(inTokenObj);
tokenObj.SetTokenStr(tokenStr);
tokenArrayList.Add(tokenObj);
}
return(tokenArrayList);
}