private static Token DigitStartString(Tokenizer t) { int i; for (i = 0; t.IsReadable(i); i++) { if (t.MatchRange(i, '0', '9')) { continue; } if (i > 0 && (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_"))) { continue; } break; } return t.TakeToken(i, TokenType.DigitStartString); }
private static Token WhiteSpace(Tokenizer t) { int i; for (i = 0; t.IsReadable(i); i++) { if (t.MatchRange(i, '\x00', '\x09') || t.MatchRange(i, '\x0B', '\x0C') || t.MatchRange(i, '\x0E', '\x20') || t.MatchAny(i, "\x7F")) { continue; } break; } return t.TakeToken(i, TokenType.WhiteSpace); }
private static Token LetterStartString(Tokenizer t) { int i; bool escape = false; for (i = 0; t.IsReadable(i); i++) { if (escape && t.MatchRange(i, '!', '~')) { escape = false; continue; } if (t.MatchRange(i, 'a', 'z') || t.MatchRange(i, 'A', 'Z') || t.MatchAny(i, "_")) { escape = false; continue; } if (i > 0 && t.MatchRange(i, '0', '9')) { escape = false; continue; } if (t.MatchAny(i, "\\")) { escape = !escape; continue; } break; } return t.TakeToken(i, TokenType.LetterStartString); }
private static Token OtherString(Tokenizer t) { int i; for (i = 0; t.IsReadable(i); i++) { if (!t.MatchRange(i, '\x00', '\x7F')) { continue; } break; } return t.TakeToken(i, TokenType.OtherString); }
public void MatchRange(string text, char start, char end, bool expected) { Tokenizer t = new Tokenizer(text, string.Empty); Assert.That(t.MatchRange(0, start, end), Is.EqualTo(expected)); }