//////////////////////////////////////////////////////////////////////////
// Construction
//////////////////////////////////////////////////////////////////////////
public HaystackTokenizer(StreamReader instrm)
{
m_srIn = instrm;
m_tok = HaystackToken.eof;
m_bInitial = true;
//consume();
//consume();
}
private void Verify(HaystackToken expected)
{
HaystackToken tokToCheck = _currentToken;
if (expected == HaystackToken.eof) // eof will stop at peek
{
tokToCheck = _peekToken;
}
if (tokToCheck != expected)
{
throw GetException("Expected " + expected + " not " + curToStr());
}
}
private void verify(HaystackToken expected)
{
HaystackToken tokToCheck = m_tokCur;
if (expected == HaystackToken.eof) // eof will stop at peek
{
tokToCheck = m_tokPeek;
}
if (tokToCheck != expected)
{
err("Expected " + expected + " not " + curToStr());
}
}
private void Consume(HaystackToken expected)
{
if (expected != null)
{
Verify(expected);
}
_currentToken = _peekToken;
_currentValue = _peekValue;
_currentLineNumber = _peekLineNumber;
_peekToken = _tokenizer.Next();
_peekValue = _tokenizer.Val;
_peekLineNumber = _tokenizer.LineNumber;
}
private void consume(HaystackToken expected)
{
if (expected != null)
{
verify(expected);
}
m_tokCur = m_tokPeek;
m_curVal = m_peekVal;
m_icurLine = m_ipeekLine;
m_tokPeek = m_tokenizer.next();
m_peekVal = m_tokenizer.Val;
m_ipeekLine = m_tokenizer.Line;
}
public bool hequals(object o)
{
if (this == o)
{
return(true); // reference check
}
if (o == null || (!(o is HaystackToken)))
{
return(false); // null and type check
}
HaystackToken that = (HaystackToken)o;
// Value compare
if (m_bliteral != that.Literal)
{
return(false);
}
return(m_strSymbol.CompareTo(that.Symbol) == 0);
}
private bool TryVerify(HaystackToken expected, bool bErr)
{
HaystackToken tokToCheck = _currentToken;
if (expected == HaystackToken.eof) // eof will stop at peek
{
tokToCheck = _peekToken;
}
if (tokToCheck != expected)
{
if (bErr)
{
throw GetException("Expected " + expected + " not " + curToStr());
}
return(false);
}
else
{
return(true);
}
}
private bool tryVerify(HaystackToken expected, bool bErr)
{
HaystackToken tokToCheck = m_tokCur;
if (expected == HaystackToken.eof) // eof will stop at peek
{
tokToCheck = m_tokPeek;
}
if (tokToCheck != expected)
{
if (bErr)
{
err("Expected " + expected + " not " + curToStr());
}
return(false);
}
else
{
return(true);
}
}
public HaystackToken Next()
{
if (_atStart)
{
consume();
consume();
_atStart = false;
}
// reset
_currentValue = null;
// skip non-meaningful whitespace and comments
int startLine;
while (true) // This is valid but not how I would implement a loop - may change to something more robust later
{
// treat space, tab, non-breaking space as whitespace
if (_currentChar != _endOfFile)
{
if ((char)_currentChar == ' ' || (char)_currentChar == '\t' || (char)_currentChar == 0xa0)
{
consume();
continue;
}
// comments
if ((char)_currentChar == '/')
{
if ((char)_peekChar == '/')
{
skipCommentsSL(); continue;
}
if ((char)_peekChar == '*')
{
skipCommentsML(); continue;
}
}
}
break;
}
if (_currentChar != _endOfFile)
{
// newlines
if ((char)_currentChar == '\n' || (char)_currentChar == '\r')
{
if ((char)_currentChar == '\r' && (char)_peekChar == '\n')
{
consume('\r');
}
consume();
++_currentLineNumber;
return(_currentToken = HaystackToken.nl);
}
}
// handle various starting chars
if (_currentChar >= 0)
{
if (isIdStart(_currentChar))
{
return(_currentToken = ReadId());
}
if (_currentChar == '"')
{
return(_currentToken = ReadString());
}
if (_currentChar == '@')
{
return(_currentToken = ReadReference());
}
if (isDigit(_currentChar))
{
return(_currentToken = ReadNumber());
}
if (_currentChar == '`')
{
return(_currentToken = ReadUri());
}
if (_currentChar == '-' && isDigit(_peekChar))
{
return(_currentToken = ReadNumber());
}
}
return(_currentToken = symbol());
}
//////////////////////////////////////////////////////////////////////////
// Tokenizing
//////////////////////////////////////////////////////////////////////////
public HaystackToken next()
{
if (m_bInitial)
{
consume();
consume();
m_bInitial = false;
}
// reset
m_val = null;
// skip non-meaningful whitespace and comments
int startLine = m_iLine;
while (true) // This is valid but not how I would implement a loop - may change to something more robust later
{
// treat space, tab, non-breaking space as whitespace
if (m_cCur != m_iEOF)
{
if ((char)m_cCur == ' ' || (char)m_cCur == '\t' || (char)m_cCur == 0xa0)
{
consume();
continue;
}
// comments
if ((char)m_cCur == '/')
{
if ((char)m_cPeek == '/')
{
skipCommentsSL(); continue;
}
if ((char)m_cPeek == '*')
{
skipCommentsML(); continue;
}
}
}
break;
}
if (m_cCur != m_iEOF)
{
// newlines
if ((char)m_cCur == '\n' || (char)m_cCur == '\r')
{
if ((char)m_cCur == '\r' && (char)m_cPeek == '\n')
{
consume('\r');
}
consume();
++m_iLine;
return(m_tok = HaystackToken.nl);
}
}
//if ((m_bCurInit) && (m_bEOF)) return m_tok = HaystackToken.eof;
// handle various starting chars
if (m_cCur >= 0)
{
if (isIdStart(m_cCur))
{
return(m_tok = id());
}
if (m_cCur == '"')
{
return(m_tok = str());
}
if (m_cCur == '@')
{
return(m_tok = refh());
}
if (isDigit(m_cCur))
{
return(m_tok = num());
}
if (m_cCur == '`')
{
return(m_tok = uri());
}
if (m_cCur == '-' && isDigit(m_cPeek))
{
return(m_tok = num());
}
}
return(m_tok = symbol());
}
