private string _Collect(CharacterClass expect, char ch) { CharacterBuffer buf = new CharacterBuffer(ch); while (true) { int cur = _reader.Peek(); if (cur == -1) { break; } ch = (char)cur; CharacterClass cc = _settings.Classifier [ch]; if ((cc & expect) == 0) { break; } buf.Write((char)_reader.Read()); } return(buf.ToString()); }
private string _CollectUntil(char[] stop, char[] disallowed) { CharacterBuffer buf = new CharacterBuffer(); while (true) { int cur = _reader.Read(); if (cur == -1) { throw new IOException(); } char ch = (char)cur; if (Array.IndexOf(stop, ch) >= 0) { break; } if (Array.IndexOf(disallowed, ch) >= 0) { throw new ApplicationException(); } buf.Write(ch); } return(buf.ToString()); }
private Token _GetToken() { Token result = new Token(); int cur = _reader.Read(); if (cur == -1) { result._characterClass = CharacterClass.EndOfFile; return(result); } char ch = (char)cur; CharacterClass cc = _settings.Classifier [ch]; CharacterBuffer cb = new CharacterBuffer(); cb.Write(ch); switch (cc) { case CharacterClass.Whitespace: result._characterClass = CharacterClass.Whitespace; result._value = _Collect(CharacterClass.Whitespace, ch); if (_settings.SkipWhiteSpace) { return(_GetToken()); } break; case CharacterClass.Quote: result._characterClass = CharacterClass.Quote; result._value = _CollectUntil (new char[] { ch }, new char[] { '\n' }); break; case CharacterClass.Comment: result._characterClass = CharacterClass.Comment; result._value = _CollectUntil (new char[] { '\n' }, new char[0]); break; case CharacterClass.Digit: result._characterClass = CharacterClass.Digit; result._value = _Collect(CharacterClass.Digit, ch); break; case CharacterClass.Punctuation: result._characterClass = CharacterClass.Punctuation; result._value = new string ( ch, 1 ); break; case CharacterClass.Word: result._characterClass = CharacterClass.Word; result._value = _Collect(CharacterClass.Word, ch); break; case CharacterClass.EndOfFile: result._characterClass = CharacterClass.EndOfFile; break; } return(result); }