public TokenObj(TokenObj inTokenObj, string tokenStr) { index_ = inTokenObj.GetIndex(); // index of this token in the TextObj pos_ = inTokenObj.GetPos(); // position tag_ = inTokenObj.GetTag(); orgTokenStr_ = inTokenObj.GetOrgTokenStr(); // the org str, never change // history of processes procHist_ = new List <string>(inTokenObj.GetProcHist()); tokenStr_ = tokenStr; }
public static List <TokenObj> FlatTokenToArrayList(TokenObj inTokenObj) { string[] tokenArray = inTokenObj.GetTokenStr().Split(patternStrSpace_, true); List <TokenObj> tokenArrayList = new List <TokenObj>(); foreach (string tokenStr in tokenArray) { TokenObj tokenObj = new TokenObj(inTokenObj); tokenObj.SetTokenStr(tokenStr); tokenArrayList.Add(tokenObj); } return(tokenArrayList); }
// update index and position for each TokenObj in the list // index: is the index of the tokenList // pos: is the index (ignoring empty string) public static void UpdateIndexPos(List <TokenObj> tokenList) { if ((tokenList != null) && (tokenList.Count > 0)) { int pos = 0; for (int index = 0; index < tokenList.Count; index++) { TokenObj tokenObj = tokenList[index]; tokenObj.SetIndex(index); if (tokenObj.IsSpaceToken() == false) { tokenObj.SetPos(pos); pos++; } } } }