internal TokenizerStream() { m_countTokens = 0; m_headTokens = new TokenizerShortBlock(); m_headStrings = new TokenizerStringBlock(); Reset(); }
internal void Reset() { m_lastTokens = null; m_currentTokens = m_headTokens; m_currentStrings = m_headStrings; m_indexTokens = 0; m_indexStrings = 0; #if _DEBUG m_bLastWasCStr = false; #endif }
internal void AddString(String str) { if (m_currentStrings.m_block.Length <= m_indexStrings) { m_currentStrings.m_next = new TokenizerStringBlock(); m_currentStrings = m_currentStrings.m_next; m_indexStrings = 0; } m_currentStrings.m_block[m_indexStrings++] = str; }
internal String GetNextString() { if (m_currentStrings.m_block.Length <= m_indexStrings) { m_currentStrings = m_currentStrings.m_next; m_indexStrings = 0; } #if _DEBUG m_bLastWasCStr = false; #endif return(m_currentStrings.m_block[m_indexStrings++]); }