public IDocument Tokenize(Stream htmlStream)
{
StreamReader reader = new StreamReader(htmlStream);
m_CurrentState = DataState.Instance;
while (m_CurrentState != null)
{
m_StateChangedExternally = false;
BaseState nextState = m_CurrentState.Process(this, reader);
if (!m_StateChangedExternally)
{
m_CurrentState = nextState;
}
}
return TreeConstruction.Instance.Document;
}
internal void SetNextState(BaseState state)
{
m_CurrentState = state;
m_StateChangedExternally = true;
}