public Lexeme Read()
{
if (m_valuableCursor < m_valuableHistory.Count)
{
int fullCursor = m_valuableHistory[m_valuableCursor++];
return(m_fullHistory[fullCursor]);
}
//m_triviaCache.Clear();
while (true)
{
//run to next stopped state
m_engine.Reset();
m_lastTokenStart = m_source.PeekLocation();
m_lastState = 0;
m_lexemeValueBuilder.Clear();
if (m_source.PeekChar() < 0)
{
//return End Of Stream token
AddHistory(new Lexeme(m_scannerInfo, m_scannerInfo.EndOfStreamState,
new SourceSpan(m_lastTokenStart, m_lastTokenStart), null));
break;
}
while (true)
{
int inputCharValue = m_source.PeekChar();
if (inputCharValue < 0)
{
//end of stream, treat as stopped
break;
}
char inputChar = (char)inputCharValue;
m_engine.Input(inputChar);
if (m_engine.IsAtStoppedState)
{
//stop immediately at a stopped state
break;
}
m_lastState = m_engine.CurrentState;
m_source.ReadChar();
m_lexemeValueBuilder.Append(inputChar);
}
//skip tokens that marked with "Skip" attribute
bool isLastSkipped = IsLastTokenSkippable();
if (isLastSkipped)
{
AddHistory(new Lexeme(m_scannerInfo, m_lastState,
new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString()), false);
}
else
{
AddHistory(new Lexeme(m_scannerInfo, m_lastState,
new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString()));
break;
}
}
int lastTokenfullCursor = m_valuableHistory[m_valuableCursor - 1];
return(m_fullHistory[lastTokenfullCursor]);
}
public Lexeme Read()
{
m_triviaCache.Clear();
bool isLastSkipped = false;
do
{
//run to next stopped state
m_engine.Reset();
m_lastTokenStart = m_source.PeekLocation();
m_lastState = 0;
m_lexemeValueBuilder.Clear();
if (m_source.PeekChar() < 0)
{
//return End Of Stream token
return(new Lexeme(m_scannerInfo, m_scannerInfo.EndOfStreamState,
new SourceSpan(m_lastTokenStart, m_lastTokenStart), null, m_triviaCache));
}
while (true)
{
int inputCharValue = m_source.PeekChar();
if (inputCharValue < 0)
{
//end of stream, treat as stopped
break;
}
char inputChar = (char)inputCharValue;
m_engine.Input(inputChar);
if (m_engine.IsAtStoppedState)
{
//stop immediately at a stopped state
break;
}
else
{
m_lastState = m_engine.CurrentState;
}
m_source.ReadChar();
m_lexemeValueBuilder.Append(inputChar);
}
//skip tokens that marked with "Skip" attribute
isLastSkipped = IsLastTokenSkippable();
if (isLastSkipped)
{
m_triviaCache.Add(new Lexeme(m_scannerInfo, m_lastState,
new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString(), null));
}
} while (isLastSkipped);
return(new Lexeme(m_scannerInfo, m_lastState,
new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString(), m_triviaCache.Count == 0 ? null : m_triviaCache));
}