private string _Collect(CharacterClass expect, char ch)
{
CharacterBuffer buf = new CharacterBuffer(ch);
while (true)
{
int cur = _reader.Peek();
if (cur == -1)
{
break;
}
ch = (char)cur;
CharacterClass cc = _settings.Classifier [ch];
if ((cc & expect) == 0)
{
break;
}
buf.Write((char)_reader.Read());
}
return(buf.ToString());
}
private string _CollectUntil(char[] stop, char[] disallowed)
{
CharacterBuffer buf = new CharacterBuffer();
while (true)
{
int cur = _reader.Read();
if (cur == -1)
{
throw new IOException();
}
char ch = (char)cur;
if (Array.IndexOf(stop, ch) >= 0)
{
break;
}
if (Array.IndexOf(disallowed, ch) >= 0)
{
throw new ApplicationException();
}
buf.Write(ch);
}
return(buf.ToString());
}
private Token _GetToken()
{
Token result = new Token();
int cur = _reader.Read();
if (cur == -1)
{
result._characterClass = CharacterClass.EndOfFile;
return(result);
}
char ch = (char)cur;
CharacterClass cc = _settings.Classifier [ch];
CharacterBuffer cb = new CharacterBuffer();
cb.Write(ch);
switch (cc)
{
case CharacterClass.Whitespace:
result._characterClass = CharacterClass.Whitespace;
result._value = _Collect(CharacterClass.Whitespace, ch);
if (_settings.SkipWhiteSpace)
{
return(_GetToken());
}
break;
case CharacterClass.Quote:
result._characterClass = CharacterClass.Quote;
result._value = _CollectUntil
(new char[] { ch },
new char[] { '\n' });
break;
case CharacterClass.Comment:
result._characterClass = CharacterClass.Comment;
result._value = _CollectUntil
(new char[] { '\n' },
new char[0]);
break;
case CharacterClass.Digit:
result._characterClass = CharacterClass.Digit;
result._value = _Collect(CharacterClass.Digit, ch);
break;
case CharacterClass.Punctuation:
result._characterClass = CharacterClass.Punctuation;
result._value = new string ( ch, 1 );
break;
case CharacterClass.Word:
result._characterClass = CharacterClass.Word;
result._value = _Collect(CharacterClass.Word, ch);
break;
case CharacterClass.EndOfFile:
result._characterClass = CharacterClass.EndOfFile;
break;
}
return(result);
}
