public TokenObj(TokenObj inTokenObj, string tokenStr)
{
index_ = inTokenObj.GetIndex(); // index of this token in the TextObj
pos_ = inTokenObj.GetPos(); // position
tag_ = inTokenObj.GetTag();
orgTokenStr_ = inTokenObj.GetOrgTokenStr(); // the org str, never change
// history of processes
procHist_ = new List <string>(inTokenObj.GetProcHist());
tokenStr_ = tokenStr;
}
public static List <TokenObj> FlatTokenToArrayList(TokenObj inTokenObj)
{
string[] tokenArray = inTokenObj.GetTokenStr().Split(patternStrSpace_, true);
List <TokenObj> tokenArrayList = new List <TokenObj>();
foreach (string tokenStr in tokenArray)
{
TokenObj tokenObj = new TokenObj(inTokenObj);
tokenObj.SetTokenStr(tokenStr);
tokenArrayList.Add(tokenObj);
}
return(tokenArrayList);
}
// update index and position for each TokenObj in the list
// index: is the index of the tokenList
// pos: is the index (ignoring empty string)
public static void UpdateIndexPos(List <TokenObj> tokenList)
{
if ((tokenList != null) && (tokenList.Count > 0))
{
int pos = 0;
for (int index = 0; index < tokenList.Count; index++)
{
TokenObj tokenObj = tokenList[index];
tokenObj.SetIndex(index);
if (tokenObj.IsSpaceToken() == false)
{
tokenObj.SetPos(pos);
pos++;
}
}
}
}
