void Init() { pos = -1; line = 1; col = 0; oldEols = 0; NextCh(); if (ch == 0xEF) { // check optional byte order mark for UTF-8 NextCh(); int ch1 = ch; NextCh(); int ch2 = ch; if (ch1 != 0xBB || ch2 != 0xBF) { throw new FatalError(String.Format("illegal byte order mark: EF {0,2:X} {1,2:X}", ch1, ch2)); } buffer = new UTF8Buffer(buffer); col = 0; NextCh(); } start = new Dictionary<int, int>(128); for (int i = 95; i <= 95; ++i) start[i] = 1; for (int i = 97; i <= 114; ++i) start[i] = 1; for (int i = 116; i <= 122; ++i) start[i] = 1; for (int i = 49; i <= 57; ++i) start[i] = 52; start[48] = 53; start[46] = 54; start[39] = 16; start[34] = 55; start[64] = 27; start[38] = 105; start[61] = 56; start[58] = 57; start[44] = 30; start[45] = 106; start[47] = 107; start[62] = 108; start[43] = 109; start[123] = 38; start[91] = 39; start[40] = 40; start[60] = 110; start[37] = 111; start[33] = 58; start[124] = 112; start[125] = 43; start[93] = 44; start[41] = 45; start[59] = 46; start[126] = 47; start[42] = 113; start[94] = 114; start[115] = 115; start[63] = 102; start[Buffer.EOF] = -1; pt = tokens = new Token(); // first token is a dummy }
Token NextToken() { while (ch == ' ' || ch >= 9 && ch <= 10 || ch == 13 ) NextCh(); int apx = 0; t = new Token(); t.pos = pos; t.col = col; t.line = line; int state; try { state = start[ch]; } catch (KeyNotFoundException) { state = 0; } tlen = 0; AddCh(); switch (state) { case -1: { t.kind = eofSym; break; } // NextCh already done case 0: { t.kind = noSym; break; } // NextCh already done case 1: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'z') { AddCh(); goto case 1; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 2: if (ch >= '0' && ch <= '9') { apx = 0; AddCh(); goto case 2; } else if (ch == 'u') { apx = 0; AddCh(); goto case 59; } else if (ch == 'l') { apx = 0; AddCh(); goto case 60; } else { tlen -= apx; buffer.Pos = t.pos; NextCh(); line = t.line; col = t.col; for (int i = 0; i < tlen; i++) NextCh(); t.kind = 2; break; } case 3: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 4; } else { t.kind = noSym; break; } case 4: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 4; } else if (ch == 'u') { AddCh(); goto case 61; } else if (ch == 'l') { AddCh(); goto case 62; } else { t.kind = 2; break; } case 5: if (ch >= '0' && ch <= '9') { AddCh(); goto case 5; } else if (ch == 'd' || ch == 'f' || ch == 'm') { AddCh(); goto case 63; } else if (ch == 'e') { AddCh(); goto case 6; } else { t.kind = 3; break; } case 6: if (ch >= '0' && ch <= '9') { AddCh(); goto case 8; } else if (ch == '+' || ch == '-') { AddCh(); goto case 7; } else { t.kind = noSym; break; } case 7: if (ch >= '0' && ch <= '9') { AddCh(); goto case 8; } else { t.kind = noSym; break; } case 8: if (ch >= '0' && ch <= '9') { AddCh(); goto case 8; } else if (ch == 'd' || ch == 'f' || ch == 'm') { AddCh(); goto case 63; } else { t.kind = 3; break; } case 9: if (ch >= '0' && ch <= '9') { AddCh(); goto case 9; } else if (ch == 'd' || ch == 'f' || ch == 'm') { AddCh(); goto case 63; } else if (ch == 'e') { AddCh(); goto case 10; } else { t.kind = 3; break; } case 10: if (ch >= '0' && ch <= '9') { AddCh(); goto case 12; } else if (ch == '+' || ch == '-') { AddCh(); goto case 11; } else { t.kind = noSym; break; } case 11: if (ch >= '0' && ch <= '9') { AddCh(); goto case 12; } else { t.kind = noSym; break; } case 12: if (ch >= '0' && ch <= '9') { AddCh(); goto case 12; } else if (ch == 'd' || ch == 'f' || ch == 'm') { AddCh(); goto case 63; } else { t.kind = 3; break; } case 13: if (ch >= '0' && ch <= '9') { AddCh(); goto case 15; } else if (ch == '+' || ch == '-') { AddCh(); goto case 14; } else { t.kind = noSym; break; } case 14: if (ch >= '0' && ch <= '9') { AddCh(); goto case 15; } else { t.kind = noSym; break; } case 15: if (ch >= '0' && ch <= '9') { AddCh(); goto case 15; } else if (ch == 'd' || ch == 'f' || ch == 'm') { AddCh(); goto case 63; } else { t.kind = 3; break; } case 16: if (ch <= 12 || ch >= 14 && ch <= '&' || ch >= '(' && ch <= '[' || ch >= ']' && ch <= 65535) { AddCh(); goto case 17; } else if (ch == 92) { AddCh(); goto case 64; } else { t.kind = noSym; break; } case 17: if (ch == 39) { AddCh(); goto case 23; } else { t.kind = noSym; break; } case 18: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 19; } else { t.kind = noSym; break; } case 19: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 65; } else if (ch == 39) { AddCh(); goto case 23; } else { t.kind = noSym; break; } case 20: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 21; } else { t.kind = noSym; break; } case 21: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 22; } else { t.kind = noSym; break; } case 22: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 17; } else { t.kind = noSym; break; } case 23: { t.kind = 4; break; } case 24: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) { AddCh(); goto case 24; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 25: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 26; } else { t.kind = noSym; break; } case 26: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= ':' && ch <= '[' || ch >= ']' && ch <= '`' || ch >= 'g' && ch <= 65535) { AddCh(); goto case 24; } else if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 68; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 27: if (ch == '"') { AddCh(); goto case 28; } else { t.kind = noSym; break; } case 28: if (ch <= '!' || ch >= '#' && ch <= 65535) { AddCh(); goto case 28; } else if (ch == '"') { AddCh(); goto case 70; } else { t.kind = noSym; break; } case 29: { t.kind = 5; break; } case 30: { t.kind = 9; break; } case 31: { t.kind = 10; break; } case 32: { t.kind = 13; break; } case 33: { t.kind = 15; break; } case 34: { t.kind = 16; break; } case 35: { t.kind = 17; break; } case 36: { t.kind = 20; break; } case 37: { t.kind = 21; break; } case 38: { t.kind = 22; break; } case 39: { t.kind = 23; break; } case 40: { t.kind = 24; break; } case 41: { t.kind = 27; break; } case 42: { t.kind = 29; break; } case 43: { t.kind = 32; break; } case 44: { t.kind = 33; break; } case 45: { t.kind = 34; break; } case 46: { t.kind = 36; break; } case 47: { t.kind = 37; break; } case 48: if (ch <= ')' || ch >= '+' && ch <= 65535) { AddCh(); goto case 48; } else if (ch == '*') { AddCh(); goto case 71; } else { t.kind = noSym; break; } case 49: { t.kind = 61; break; } case 50: if (ch == 13) { AddCh(); goto case 51; } else if (ch <= 12 || ch >= 14 && ch <= 65535) { AddCh(); goto case 50; } else { t.kind = noSym; break; } case 51: { t.kind = 62; break; } case 52: if (ch >= '0' && ch <= '9') { apx = 0; AddCh(); goto case 52; } else if (ch == 'u') { apx = 0; AddCh(); goto case 59; } else if (ch == 'l') { apx = 0; AddCh(); goto case 60; } else if (ch == '.') { apx++; AddCh(); goto case 72; } else if (ch == 'e') { apx = 0; AddCh(); goto case 13; } else if (ch == 'd' || ch == 'f' || ch == 'm') { apx = 0; AddCh(); goto case 63; } else { t.kind = 2; break; } case 53: if (ch >= '0' && ch <= '9') { apx = 0; AddCh(); goto case 52; } else if (ch == 'u') { apx = 0; AddCh(); goto case 59; } else if (ch == 'l') { apx = 0; AddCh(); goto case 60; } else if (ch == '.') { apx++; AddCh(); goto case 72; } else if (ch == 'x') { apx = 0; AddCh(); goto case 3; } else if (ch == 'e') { apx = 0; AddCh(); goto case 13; } else if (ch == 'd' || ch == 'f' || ch == 'm') { apx = 0; AddCh(); goto case 63; } else { t.kind = 2; break; } case 54: if (ch >= '0' && ch <= '9') { AddCh(); goto case 5; } else { t.kind = 12; break; } case 55: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) { AddCh(); goto case 24; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = 63; break; } case 56: if (ch == '=') { AddCh(); goto case 32; } else { t.kind = 7; break; } case 57: if (ch == '=') { AddCh(); goto case 35; } else if (ch == '>') { AddCh(); goto case 73; } else if (ch == '<') { AddCh(); goto case 74; } else { t.kind = 8; break; } case 58: if (ch == '=') { AddCh(); goto case 42; } else { t.kind = 30; break; } case 59: if (ch == 'l') { AddCh(); goto case 75; } else { t.kind = 2; break; } case 60: if (ch == 'u') { AddCh(); goto case 75; } else { t.kind = 2; break; } case 61: if (ch == 'l') { AddCh(); goto case 75; } else { t.kind = 2; break; } case 62: if (ch == 'u') { AddCh(); goto case 75; } else { t.kind = 2; break; } case 63: { t.kind = 3; break; } case 64: if (ch == '"' || ch == 39 || ch == '0' || ch == 92 || ch >= 'a' && ch <= 'b' || ch == 'f' || ch == 'n' || ch == 'r' || ch == 't' || ch == 'v') { AddCh(); goto case 17; } else if (ch == 'x') { AddCh(); goto case 18; } else if (ch == 'u') { AddCh(); goto case 76; } else { t.kind = noSym; break; } case 65: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 66; } else if (ch == 39) { AddCh(); goto case 23; } else { t.kind = noSym; break; } case 66: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 17; } else if (ch == 39) { AddCh(); goto case 23; } else { t.kind = noSym; break; } case 67: if (ch == '"' || ch == 39 || ch == '0' || ch == 92 || ch >= 'a' && ch <= 'b' || ch == 'f' || ch == 'n' || ch == 'r' || ch == 't' || ch == 'v') { AddCh(); goto case 24; } else if (ch == 'x') { AddCh(); goto case 25; } else if (ch == 'u') { AddCh(); goto case 77; } else { t.kind = noSym; break; } case 68: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 69; } else if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= ':' && ch <= '[' || ch >= ']' && ch <= '`' || ch >= 'g' && ch <= 65535) { AddCh(); goto case 24; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 69: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) { AddCh(); goto case 24; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 70: if (ch == '"') { AddCh(); goto case 28; } else { t.kind = 5; break; } case 71: if (ch <= ')' || ch >= '+' && ch <= '.' || ch >= '0' && ch <= 65535) { AddCh(); goto case 48; } else if (ch == '/') { AddCh(); goto case 49; } else if (ch == '*') { AddCh(); goto case 71; } else { t.kind = noSym; break; } case 72: if (ch <= '/' || ch >= ':' && ch <= 65535) { apx++; AddCh(); goto case 2; } else if (ch >= '0' && ch <= '9') { apx = 0; AddCh(); goto case 9; } else { t.kind = noSym; break; } case 73: if (ch == '=') { AddCh(); goto case 37; } else { t.kind = 18; break; } case 74: if (ch == '=') { AddCh(); goto case 36; } else { t.kind = 19; break; } case 75: { t.kind = 2; break; } case 76: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 78; } else { t.kind = noSym; break; } case 77: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 79; } else { t.kind = noSym; break; } case 78: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 80; } else { t.kind = noSym; break; } case 79: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 81; } else { t.kind = noSym; break; } case 80: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 82; } else { t.kind = noSym; break; } case 81: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 83; } else { t.kind = noSym; break; } case 82: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 84; } else { t.kind = noSym; break; } case 83: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 85; } else { t.kind = noSym; break; } case 84: if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 20; } else if (ch == 39) { AddCh(); goto case 23; } else { t.kind = noSym; break; } case 85: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= ':' && ch <= '[' || ch >= ']' && ch <= '`' || ch >= 'g' && ch <= 65535) { AddCh(); goto case 24; } else if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 86; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 86: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= ':' && ch <= '[' || ch >= ']' && ch <= '`' || ch >= 'g' && ch <= 65535) { AddCh(); goto case 24; } else if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 87; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 87: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= ':' && ch <= '[' || ch >= ']' && ch <= '`' || ch >= 'g' && ch <= 65535) { AddCh(); goto case 24; } else if (ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f') { AddCh(); goto case 88; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 88: if (ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) { AddCh(); goto case 24; } else if (ch == '"') { AddCh(); goto case 29; } else if (ch == 92) { AddCh(); goto case 67; } else { t.kind = noSym; break; } case 89: { t.kind = 74; break; } case 90: if (ch == ']') { AddCh(); goto case 91; } else { t.kind = noSym; break; } case 91: { t.kind = 79; break; } case 92: { t.kind = 86; break; } case 93: { t.kind = 87; break; } case 94: { t.kind = 88; break; } case 95: { t.kind = 89; break; } case 96: { t.kind = 90; break; } case 97: { t.kind = 91; break; } case 98: { t.kind = 92; break; } case 99: { t.kind = 93; break; } case 100: { t.kind = 94; break; } case 101: { t.kind = 95; break; } case 102: { t.kind = 96; break; } case 103: { t.kind = 97; break; } case 104: { t.kind = 98; break; } case 105: if (ch == '=') { AddCh(); goto case 97; } else if (ch == '&') { AddCh(); goto case 104; } else { t.kind = 6; break; } case 106: if (ch == '-') { AddCh(); goto case 31; } else if (ch == '=') { AddCh(); goto case 93; } else { t.kind = 70; break; } case 107: if (ch == '*') { AddCh(); goto case 48; } else if (ch == '/') { AddCh(); goto case 50; } else if (ch == '=') { AddCh(); goto case 95; } else { t.kind = 11; break; } case 108: if (ch == '=') { AddCh(); goto case 33; } else if (ch == '>') { AddCh(); goto case 116; } else { t.kind = 14; break; } case 109: if (ch == '+') { AddCh(); goto case 34; } else if (ch == '=') { AddCh(); goto case 92; } else { t.kind = 69; break; } case 110: if (ch == '<') { AddCh(); goto case 117; } else if (ch == '=') { AddCh(); goto case 41; } else if (ch == '>') { AddCh(); goto case 89; } else { t.kind = 26; break; } case 111: if (ch == '=') { AddCh(); goto case 96; } else { t.kind = 28; break; } case 112: if (ch == '=') { AddCh(); goto case 98; } else if (ch == '|') { AddCh(); goto case 103; } else { t.kind = 31; break; } case 113: if (ch == '=') { AddCh(); goto case 94; } else { t.kind = 38; break; } case 114: if (ch == '=') { AddCh(); goto case 99; } else { t.kind = 68; break; } case 115: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 's' || ch >= 'u' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == 't') { AddCh(); goto case 118; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 116: if (ch == '=') { AddCh(); goto case 101; } else { t.kind = 35; break; } case 117: if (ch == '=') { AddCh(); goto case 100; } else { t.kind = 25; break; } case 118: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'q' || ch >= 's' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == 'r') { AddCh(); goto case 119; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 119: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'h' || ch >= 'j' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == 'i') { AddCh(); goto case 120; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 120: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'm' || ch >= 'o' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == 'n') { AddCh(); goto case 121; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 121: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'f' || ch >= 'h' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == 'g') { AddCh(); goto case 122; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } case 122: if (ch == '-' || ch >= '0' && ch <= '9' || ch == '_' || ch >= 'a' && ch <= 'z') { AddCh(); goto case 1; } else if (ch == '[') { AddCh(); goto case 90; } else { t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; } } t.val = new String(tval, 0, tlen); return t; }
// get the next token (possibly a token already seen during peeking) public Token Scan() { if (tokens.next == null) { return NextToken(); } else { pt = tokens = tokens.next; return tokens; } }
// make sure that peeking starts at the current scan position public void ResetPeek() { pt = tokens; }
// peek for the next token, ignore pragmas public Token Peek() { if (pt.next == null) { do { pt = pt.next = NextToken(); } while (pt.kind > maxT); // skip pragmas } else { do { pt = pt.next; } while (pt.kind > maxT); } return pt; }
void Get() { for (; ; ) { t = la; la = scanner.Scan(); if (la.kind <= maxT) { ++errDist; break; } la = t; } }
public void Parse() { la = new Token(); la.val = ""; Get(); Formula(); Expect(0); }