internal TokenizerStream()
{
this.m_countTokens = 0;
this.m_headTokens = new TokenizerShortBlock();
this.m_headStrings = new TokenizerStringBlock();
this.Reset();
}
internal void Reset()
{
this.m_lastTokens = (TokenizerShortBlock)null;
this.m_currentTokens = this.m_headTokens;
this.m_currentStrings = this.m_headStrings;
this.m_indexTokens = 0;
this.m_indexStrings = 0;
}
internal string GetNextString()
{
if (this.m_currentStrings.m_block.Length <= this.m_indexStrings)
{
this.m_currentStrings = this.m_currentStrings.m_next;
this.m_indexStrings = 0;
}
string[] strArray = this.m_currentStrings.m_block;
int num = this.m_indexStrings;
this.m_indexStrings = num + 1;
int index = num;
return(strArray[index]);
}
internal void AddString(string str)
{
if (this.m_currentStrings.m_block.Length <= this.m_indexStrings)
{
this.m_currentStrings.m_next = new TokenizerStringBlock();
this.m_currentStrings = this.m_currentStrings.m_next;
this.m_indexStrings = 0;
}
string[] strArray = this.m_currentStrings.m_block;
int num = this.m_indexStrings;
this.m_indexStrings = num + 1;
int index = num;
string str1 = str;
strArray[index] = str1;
}
