internal TokenizerStream()
{
this.m_countTokens = 0;
this.m_headTokens = new TokenizerShortBlock();
this.m_headStrings = new TokenizerStringBlock();
this.Reset();
}
internal void Reset()
{
this.m_lastTokens = (TokenizerShortBlock)null;
this.m_currentTokens = this.m_headTokens;
this.m_currentStrings = this.m_headStrings;
this.m_indexTokens = 0;
this.m_indexStrings = 0;
}
internal short GetNextFullToken()
{
if (this.m_currentTokens.m_block.Length <= this.m_indexTokens)
{
this.m_lastTokens = this.m_currentTokens;
this.m_currentTokens = this.m_currentTokens.m_next;
this.m_indexTokens = 0;
}
short[] numArray = this.m_currentTokens.m_block;
int num = this.m_indexTokens;
this.m_indexTokens = num + 1;
int index = num;
return(numArray[index]);
}
internal void AddToken(short token)
{
if (this.m_currentTokens.m_block.Length <= this.m_indexTokens)
{
this.m_currentTokens.m_next = new TokenizerShortBlock();
this.m_currentTokens = this.m_currentTokens.m_next;
this.m_indexTokens = 0;
}
this.m_countTokens = this.m_countTokens + 1;
short[] numArray = this.m_currentTokens.m_block;
int num1 = this.m_indexTokens;
this.m_indexTokens = num1 + 1;
int index = num1;
int num2 = (int)token;
numArray[index] = (short)num2;
}
