コード例 #1
0
 // get the next token (possibly a token already seen during peeking)
 public Token Scan()
 {
     if (tokens.next == null) {
         return NextToken();
     } else {
         pt = tokens = tokens.next;
         return tokens;
     }
 }
コード例 #2
0
 void Init()
 {
     pos = -1; line = 1; col = 0;
     oldEols = 0;
     NextCh();
     if (ch == 0xEF) { // check optional byte order mark for UTF-8
         NextCh(); int ch1 = ch;
         NextCh(); int ch2 = ch;
         if (ch1 != 0xBB || ch2 != 0xBF) {
             throw new FatalError(String.Format("illegal byte order mark: EF {0,2:X} {1,2:X}", ch1, ch2));
         }
         buffer = new UTF8Buffer(buffer); col = 0;
         NextCh();
     }
     pt = tokens = new Token();  // first token is a dummy
 }
コード例 #3
0
        // peek for the next token, ignore pragmas
        public Token Peek()
        {
            do {
                if (pt.next == null) {
                    pt.next = NextToken();
                }
                pt = pt.next;
            } while (pt.kind > maxT); // skip pragmas

            return pt;
        }
コード例 #4
0
 // make sure that peeking starts at the current scan position
 public void ResetPeek()
 {
     pt = tokens;
 }
コード例 #5
0
        Token NextToken()
        {
            while (ch == ' ' ||
            ch >= 9 && ch <= 10 || ch == 13
            ) NextCh();
            if (ch == '#' && Comment0()) return NextToken();
            t = new Token();
            t.pos = pos; t.col = col; t.line = line;
            int state;
            if (start.ContainsKey(ch)) { state = (int) start[ch]; }
            else { state = 0; }
            tlen = 0; AddCh();

            switch (state) {
            case -1: { t.kind = eofSym; break; } // NextCh already done
            case 0: { t.kind = noSym; break; }   // NextCh already done
            case 1:
                {t.kind = 2; break;}
            case 2:
                if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 2;}
                else {t.kind = 3; t.val = new String(tval, 0, tlen); CheckLiteral(); return t;}
            case 3:
                if (ch >= 'A' && ch <= 'Z') {AddCh(); goto case 4;}
                else {t.kind = noSym; break;}
            case 4:
                if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 4;}
                else if (ch == '.') {AddCh(); goto case 5;}
                else {t.kind = 4; break;}
            case 5:
                if (ch >= 'A' && ch <= 'Z') {AddCh(); goto case 6;}
                else {t.kind = noSym; break;}
            case 6:
                if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 6;}
                else if (ch == '.') {AddCh(); goto case 5;}
                else {t.kind = 4; break;}
            case 7:
                if (ch >= 'a' && ch <= 'z') {AddCh(); goto case 8;}
                else {t.kind = noSym; break;}
            case 8:
                if (ch == '_') {AddCh(); goto case 9;}
                else if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 8;}
                else {t.kind = noSym; break;}
            case 9:
                {t.kind = 5; break;}
            case 10:
                if (ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 11;}
                else {t.kind = noSym; break;}
            case 11:
                if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 11;}
                else {t.kind = 6; break;}
            case 12:
                if (ch >= '0' && ch <= '9') {AddCh(); goto case 12;}
                else {t.kind = 7; t.val = new String(tval, 0, tlen); CheckLiteral(); return t;}
            case 13:
                if (ch <= '!' || ch >= '#' && ch <= 65535) {AddCh(); goto case 13;}
                else if (ch == '"') {AddCh(); goto case 14;}
                else {t.kind = noSym; break;}
            case 14:
                {t.kind = 8; break;}
            case 15:
                if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); goto case 15;}
                else if (ch == 39) {AddCh(); goto case 16;}
                else if (ch == '_') {AddCh(); goto case 1;}
                else if (ch == '.') {AddCh(); goto case 3;}
                else {t.kind = 1; break;}
            case 16:
                if (ch == 39) {AddCh(); goto case 16;}
                else if (ch == '_') {AddCh(); goto case 1;}
                else {t.kind = 1; break;}
            case 17:
                {t.kind = 10; break;}
            case 18:
                {t.kind = 11; break;}
            case 19:
                {t.kind = 12; break;}
            case 20:
                {t.kind = 14; break;}
            case 21:
                {t.kind = 15; break;}
            case 22:
                {t.kind = 16; break;}
            case 23:
                {t.kind = 21; break;}
            case 24:
                {t.kind = 22; break;}
            case 25:
                {t.kind = 23; break;}
            case 26:
                {t.kind = 24; break;}
            case 27:
                {t.kind = 25; break;}
            case 28:
                {t.kind = 26; break;}
            case 29:
                {t.kind = 30; break;}
            case 30:
                if (ch == '=') {AddCh(); goto case 31;}
                else {t.kind = noSym; break;}
            case 31:
                {t.kind = 31; break;}
            case 32:
                {t.kind = 33; break;}
            case 33:
                {t.kind = 35; break;}
            case 34:
                {t.kind = 36; break;}
            case 35:
                {t.kind = 37; break;}
            case 36:
                {t.kind = 38; break;}
            case 37:
                if (ch == '=') {AddCh(); goto case 29;}
                else {t.kind = 13; break;}
            case 38:
                if (ch == '=') {AddCh(); goto case 32;}
                else {t.kind = 32; break;}
            case 39:
                if (ch == '=') {AddCh(); goto case 33;}
                else {t.kind = 34; break;}

            }
            t.val = new String(tval, 0, tlen);
            return t;
        }
コード例 #6
0
        public void Parse()
        {
            la = new Token();
            la.val = "";
            Get();
            CCS();

            Expect(0);
        }
コード例 #7
0
 private void SetStartPos(Node n, Token t)
 {
     n.LexicalInfo.StartLine = t.line;
     n.LexicalInfo.StartColumn = t.col;
 }
コード例 #8
0
 private void SetPos(Node n, Token start, Token end)
 {
     n.LexicalInfo.StartLine = start.line;
     n.LexicalInfo.StartColumn = start.col;
     n.LexicalInfo.EndLine = end.line;
     n.LexicalInfo.EndColumn = end.col + end.val.Length;
 }
コード例 #9
0
 private void SetPos(Node n, Token t)
 {
     n.LexicalInfo.StartLine = t.line;
     n.LexicalInfo.StartColumn = t.col;
     n.LexicalInfo.EndLine = t.line;
     n.LexicalInfo.EndColumn = t.col + t.val.Length;
 }
コード例 #10
0
 private void SetEndPos(Node n, Token t)
 {
     n.LexicalInfo.EndLine = t.line;
     n.LexicalInfo.EndColumn = t.col;
 }
コード例 #11
0
 private void CopyPos(Node n, Node source, Token end)
 {
     n.LexicalInfo.StartLine = source.LexicalInfo.StartLine;
     n.LexicalInfo.StartColumn = source.LexicalInfo.StartColumn;
     n.LexicalInfo.EndLine = end.line;
     n.LexicalInfo.EndColumn = end.col + end.val.Length;
 }