private static Token DigitStartString(Tokenizer t)
{
int i;
for (i = 0; t.IsReadable(i); i++)
{
if (t.MatchRange(i, '0', '9'))
{
continue;
}
if (i > 0 && (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_")))
{
continue;
}
break;
}
return t.TakeToken(i, TokenType.DigitStartString);
}
private static Token WhiteSpace(Tokenizer t)
{
int i;
for (i = 0; t.IsReadable(i); i++)
{
if (t.MatchRange(i, '\x00', '\x09') || t.MatchRange(i, '\x0B', '\x0C') || t.MatchRange(i, '\x0E', '\x20') || t.MatchAny(i, "\x7F"))
{
continue;
}
break;
}
return t.TakeToken(i, TokenType.WhiteSpace);
}
private static Token LetterStartString(Tokenizer t)
{
int i;
bool escape = false;
for (i = 0; t.IsReadable(i); i++)
{
if (escape && t.MatchRange(i, '!', '~'))
{
escape = false;
continue;
}
if (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_"))
{
escape = false;
continue;
}
if (i > 0 && t.MatchRange(i, '0', '9'))
{
escape = false;
continue;
}
if (t.MatchAny(i, "\\"))
{
escape = !escape;
continue;
}
break;
}
return t.TakeToken(i, TokenType.LetterStartString);
}
private static Token OtherString(Tokenizer t)
{
int i;
for (i = 0; t.IsReadable(i); i++)
{
if (!t.MatchRange(i, '\x00', '\x7F'))
{
continue;
}
break;
}
return t.TakeToken(i, TokenType.OtherString);
}
public void MatchRange(string text, char start, char end, bool expected)
{
Tokenizer t = new Tokenizer(text, string.Empty);
Assert.That(t.MatchRange(0, start, end), Is.EqualTo(expected));
}
