// FreeToken - return a token to the spare token pool
internal void FreeToken(Token token)
{
token.data = null;
m_SpareTokens.Push(token);
}
// Split mark token
public Token SplitMarkToken(List<Token> tokens, List<Token> marks, Token token, int position)
{
// Create the new rhs token
Token tokenRhs = CreateToken(token.type, token.startOffset + position, token.length - position);
// Adjust down the length of this token
token.length = position;
// Insert the new token into each of the parent collections
marks.Insert(marks.IndexOf(token) + 1, tokenRhs);
tokens.Insert(tokens.IndexOf(token) + 1, tokenRhs);
// Return the new token
return tokenRhs;
}
