////////////////////////////////////////////////////////////////////////// // Construction ////////////////////////////////////////////////////////////////////////// public HaystackTokenizer(StreamReader instrm) { m_srIn = instrm; m_tok = HaystackToken.eof; m_bInitial = true; //consume(); //consume(); }
private void Verify(HaystackToken expected) { HaystackToken tokToCheck = _currentToken; if (expected == HaystackToken.eof) // eof will stop at peek { tokToCheck = _peekToken; } if (tokToCheck != expected) { throw GetException("Expected " + expected + " not " + curToStr()); } }
private void verify(HaystackToken expected) { HaystackToken tokToCheck = m_tokCur; if (expected == HaystackToken.eof) // eof will stop at peek { tokToCheck = m_tokPeek; } if (tokToCheck != expected) { err("Expected " + expected + " not " + curToStr()); } }
private void Consume(HaystackToken expected) { if (expected != null) { Verify(expected); } _currentToken = _peekToken; _currentValue = _peekValue; _currentLineNumber = _peekLineNumber; _peekToken = _tokenizer.Next(); _peekValue = _tokenizer.Val; _peekLineNumber = _tokenizer.LineNumber; }
private void consume(HaystackToken expected) { if (expected != null) { verify(expected); } m_tokCur = m_tokPeek; m_curVal = m_peekVal; m_icurLine = m_ipeekLine; m_tokPeek = m_tokenizer.next(); m_peekVal = m_tokenizer.Val; m_ipeekLine = m_tokenizer.Line; }
public bool hequals(object o) { if (this == o) { return(true); // reference check } if (o == null || (!(o is HaystackToken))) { return(false); // null and type check } HaystackToken that = (HaystackToken)o; // Value compare if (m_bliteral != that.Literal) { return(false); } return(m_strSymbol.CompareTo(that.Symbol) == 0); }
private bool TryVerify(HaystackToken expected, bool bErr) { HaystackToken tokToCheck = _currentToken; if (expected == HaystackToken.eof) // eof will stop at peek { tokToCheck = _peekToken; } if (tokToCheck != expected) { if (bErr) { throw GetException("Expected " + expected + " not " + curToStr()); } return(false); } else { return(true); } }
private bool tryVerify(HaystackToken expected, bool bErr) { HaystackToken tokToCheck = m_tokCur; if (expected == HaystackToken.eof) // eof will stop at peek { tokToCheck = m_tokPeek; } if (tokToCheck != expected) { if (bErr) { err("Expected " + expected + " not " + curToStr()); } return(false); } else { return(true); } }
public HaystackToken Next() { if (_atStart) { consume(); consume(); _atStart = false; } // reset _currentValue = null; // skip non-meaningful whitespace and comments int startLine; while (true) // This is valid but not how I would implement a loop - may change to something more robust later { // treat space, tab, non-breaking space as whitespace if (_currentChar != _endOfFile) { if ((char)_currentChar == ' ' || (char)_currentChar == '\t' || (char)_currentChar == 0xa0) { consume(); continue; } // comments if ((char)_currentChar == '/') { if ((char)_peekChar == '/') { skipCommentsSL(); continue; } if ((char)_peekChar == '*') { skipCommentsML(); continue; } } } break; } if (_currentChar != _endOfFile) { // newlines if ((char)_currentChar == '\n' || (char)_currentChar == '\r') { if ((char)_currentChar == '\r' && (char)_peekChar == '\n') { consume('\r'); } consume(); ++_currentLineNumber; return(_currentToken = HaystackToken.nl); } } // handle various starting chars if (_currentChar >= 0) { if (isIdStart(_currentChar)) { return(_currentToken = ReadId()); } if (_currentChar == '"') { return(_currentToken = ReadString()); } if (_currentChar == '@') { return(_currentToken = ReadReference()); } if (isDigit(_currentChar)) { return(_currentToken = ReadNumber()); } if (_currentChar == '`') { return(_currentToken = ReadUri()); } if (_currentChar == '-' && isDigit(_peekChar)) { return(_currentToken = ReadNumber()); } } return(_currentToken = symbol()); }
////////////////////////////////////////////////////////////////////////// // Tokenizing ////////////////////////////////////////////////////////////////////////// public HaystackToken next() { if (m_bInitial) { consume(); consume(); m_bInitial = false; } // reset m_val = null; // skip non-meaningful whitespace and comments int startLine = m_iLine; while (true) // This is valid but not how I would implement a loop - may change to something more robust later { // treat space, tab, non-breaking space as whitespace if (m_cCur != m_iEOF) { if ((char)m_cCur == ' ' || (char)m_cCur == '\t' || (char)m_cCur == 0xa0) { consume(); continue; } // comments if ((char)m_cCur == '/') { if ((char)m_cPeek == '/') { skipCommentsSL(); continue; } if ((char)m_cPeek == '*') { skipCommentsML(); continue; } } } break; } if (m_cCur != m_iEOF) { // newlines if ((char)m_cCur == '\n' || (char)m_cCur == '\r') { if ((char)m_cCur == '\r' && (char)m_cPeek == '\n') { consume('\r'); } consume(); ++m_iLine; return(m_tok = HaystackToken.nl); } } //if ((m_bCurInit) && (m_bEOF)) return m_tok = HaystackToken.eof; // handle various starting chars if (m_cCur >= 0) { if (isIdStart(m_cCur)) { return(m_tok = id()); } if (m_cCur == '"') { return(m_tok = str()); } if (m_cCur == '@') { return(m_tok = refh()); } if (isDigit(m_cCur)) { return(m_tok = num()); } if (m_cCur == '`') { return(m_tok = uri()); } if (m_cCur == '-' && isDigit(m_cPeek)) { return(m_tok = num()); } } return(m_tok = symbol()); }