internal TokenizerStream() { this.m_countTokens = 0; this.m_headTokens = new TokenizerShortBlock(); this.m_headStrings = new TokenizerStringBlock(); this.Reset(); }
internal void Reset() { this.m_lastTokens = (TokenizerShortBlock)null; this.m_currentTokens = this.m_headTokens; this.m_currentStrings = this.m_headStrings; this.m_indexTokens = 0; this.m_indexStrings = 0; }
internal short GetNextFullToken() { if (this.m_currentTokens.m_block.Length <= this.m_indexTokens) { this.m_lastTokens = this.m_currentTokens; this.m_currentTokens = this.m_currentTokens.m_next; this.m_indexTokens = 0; } short[] numArray = this.m_currentTokens.m_block; int num = this.m_indexTokens; this.m_indexTokens = num + 1; int index = num; return(numArray[index]); }
internal void AddToken(short token) { if (this.m_currentTokens.m_block.Length <= this.m_indexTokens) { this.m_currentTokens.m_next = new TokenizerShortBlock(); this.m_currentTokens = this.m_currentTokens.m_next; this.m_indexTokens = 0; } this.m_countTokens = this.m_countTokens + 1; short[] numArray = this.m_currentTokens.m_block; int num1 = this.m_indexTokens; this.m_indexTokens = num1 + 1; int index = num1; int num2 = (int)token; numArray[index] = (short)num2; }