private static Token GetToken() { States curstate; char curchar; Token curtoken = new Token(); bool savechar; bool dotflag = false; curtoken.TokenValue = ""; curstate = States.S_START; while(curstate != States.S_DONE){ curchar = GetNextCharacter(); savechar = true; switch(curstate){ case States.S_START: if(IsWhiteSpace(curchar)) { savechar = false; } else if (IsDigit(curchar)) { curtoken.startpointer = pointer-1; curstate = States.S_NUMBER; } else if (curchar=='.') { curtoken.startpointer = pointer-1; curstate = States.S_NUMBER; dotflag = true; } else if (curchar=='#') { curtoken.startpointer = pointer-1; curstate = States.S_DIRECTIVE; } else if (curchar=='='){ curtoken.startpointer = pointer-1; curstate = States.S_ISEQUAL; } else if (curchar=='>') { curtoken.startpointer = pointer-1; curstate = States.S_GREATORTHAN; } else if (curchar=='<') { curtoken.startpointer = pointer-1; curstate = States.S_LESSTHAN; } else if (curchar=='!') { curtoken.startpointer = pointer-1; curstate = States.S_NOTEQUAL; } else if (curchar=='&') { curtoken.startpointer = pointer-1; curstate = States.S_AND; } else if (curchar=='|') { curtoken.startpointer = pointer-1; curstate = States.S_OR; } else if (curchar=='/') { curtoken.startpointer = pointer-1; curstate = States.S_COMMENT; } else if (curchar=='+') { curtoken.startpointer = pointer-1; curstate = States.S_INCREMENT; } else if (curchar=='-') { curtoken.startpointer = pointer-1; curstate = States.S_DECREMENT; } else if (IsLetter(curchar)) { curtoken.startpointer = pointer-1; curstate = States.S_IDENTIFIER; } else if (curchar=='"') { curtoken.startpointer = pointer-1; curstate = States.S_LITERAL; savechar = false; } else { curtoken.startpointer = pointer-1; curtoken.endpointer = pointer; curstate=States.S_DONE; switch(curchar) { //case '+': //case '-': case '*': case '^': case '%': curtoken.TType = TokenType.OPERATOR; break; case ':': curtoken.TType = TokenType.COLON; break; case ';': curtoken.TType = TokenType.SEMICOLON; break; case '(': curtoken.TType = TokenType.LPARAN; break; case ')': curtoken.TType = TokenType.RPARAN; break; case '[': curtoken.TType = TokenType.LSQUARE; break; case ']': curtoken.TType = TokenType.RSQUARE; break; case '{': curtoken.TType = TokenType.LCURBRACKET; break; case '}': curtoken.TType = TokenType.RCURBRACKET; break; case ',': curtoken.TType = TokenType.COMMA; break; case (char)0: curtoken.TType = TokenType.END; break; default: curtoken.TType = TokenType.ERROR; break; } } break; case States.S_LITERAL: if(curchar=='"') { savechar = false; curtoken.endpointer = pointer; curtoken.TType = TokenType.LITERAL; curstate = States.S_DONE; } else if(curchar==0){ UnGetNextCharacter(); savechar = false; curtoken.endpointer = pointer; curtoken.TType = TokenType.ERROR; curstate = States.S_DONE; } break; case States.S_AND: if(curchar=='&') curtoken.TType = TokenType.LOGICAL_OPERATOR; else { UnGetNextCharacter(); savechar = false; curtoken.TType = TokenType.ERROR; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_OR: if(curchar=='|') curtoken.TType = TokenType.LOGICAL_OPERATOR; else { UnGetNextCharacter(); savechar = false; curtoken.TType = TokenType.ERROR; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_ISEQUAL: if(curchar=='=') curtoken.TType = TokenType.REL_OPERATOR; else { UnGetNextCharacter(); savechar = false; curtoken.TType = TokenType.OPERATOR; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_NOTEQUAL: if(curchar=='=') curtoken.TType = TokenType.REL_OPERATOR; else { UnGetNextCharacter(); savechar = false; curtoken.TType = TokenType.LOGICAL_OPERATOR; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_LESSTHAN: curtoken.TType = TokenType.REL_OPERATOR; if(curchar!='='){ UnGetNextCharacter(); savechar = false; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_GREATORTHAN: curtoken.TType = TokenType.REL_OPERATOR; if(curchar!='=') { UnGetNextCharacter(); savechar = false; } curtoken.endpointer = pointer; curstate = States.S_DONE; break; case States.S_INCREMENT: if(curchar=='+') curtoken.TType = TokenType.INCREMENTDECREMENT; else { savechar=false; UnGetNextCharacter(); curtoken.TType = TokenType.OPERATOR; } curstate = States.S_DONE; curtoken.endpointer = pointer; break; case States.S_DECREMENT: if(curchar=='-') curtoken.TType = TokenType.INCREMENTDECREMENT; else { savechar=false; UnGetNextCharacter(); curtoken.TType = TokenType.OPERATOR; } curstate = States.S_DONE; curtoken.endpointer = pointer; break; case States.S_COMMENT: savechar = false; if(curchar=='/') curstate = States.S_COMMENTLINE; else if(curchar=='*') curstate = States.S_COMMENTBLOCK; else{ UnGetNextCharacter(); curstate = States.S_DONE; curtoken.TType = TokenType.OPERATOR; curtoken.endpointer = pointer; } break; case States.S_COMMENTLINE: savechar=false; if(curchar==13 || curchar==0 || curchar==10){ curtoken.TokenValue = "Commented Line"; curtoken.TType = TokenType.COMMENTS; curstate = States.S_DONE; curtoken.endpointer = pointer; } break; case States.S_COMMENTBLOCK: savechar=false; if(curchar=='*') curstate = States.S_COMMENTBLOCKEND; else if(curchar==0){ curstate = States.S_DONE; curtoken.TokenValue = "Commented Block"; curtoken.TType = TokenType.COMMENTS; curtoken.endpointer = pointer; } break; case States.S_COMMENTBLOCKEND: savechar=false; if(curchar=='/') { curtoken.TokenValue = "Commented Block"; curstate = States.S_DONE; curtoken.TType = TokenType.COMMENTS; curtoken.endpointer = pointer; }else curstate = States.S_COMMENTBLOCK; break; case States.S_IDENTIFIER: if(!IsDigit(curchar) && !IsLetter(curchar)){ UnGetNextCharacter(); savechar =false; curstate = States.S_DONE; curtoken.endpointer = pointer; curtoken.TType = TokenType.IDENTIFIER; } break; case States.S_DIRECTIVE: if(!IsLetter(curchar)){ if(IsWhiteSpace(curchar)) { UnGetNextCharacter(); savechar =false; curtoken.TType = TokenType.DIRECTIVE; } else curtoken.TType = TokenType.ERROR; curstate = States.S_DONE; curtoken.endpointer = pointer; } break; case States.S_NUMBER: if(!IsDigit(curchar)) { // if(IsWhiteSpace(curchar) || curchar == ':' || curchar==')' || curchar==']' || curchar==0 || curchar==',' || curchar=='}') // { if(curchar=='.') { if(dotflag==false) dotflag=true; else { curstate = States.S_DONE; curtoken.TType = TokenType.ERROR; curtoken.endpointer = pointer; } } else { UnGetNextCharacter(); savechar =false; curstate = States.S_DONE; curtoken.TType = TokenType.NUMBER; curtoken.endpointer = pointer; } // } // else{ // UnGetNextCharacter(); // savechar =false; // curstate = States.S_DONE; // curtoken.TType = TokenType.ERROR; // } } break; } if(savechar)curtoken.TokenValue = curtoken.TokenValue+curchar; } if(curtoken.TType == TokenType.IDENTIFIER) { if(ReserveWords.IndexOf(","+curtoken.TokenValue+",")!=-1) curtoken.TType = TokenType.FUNCTION; else if(KeyWords.IndexOf(","+curtoken.TokenValue+",")!=-1) curtoken.TType = TokenType.KEYWORD; else if(Procedures.IndexOf(","+curtoken.TokenValue+",")!=-1) curtoken.TType = TokenType.PROCEDURE; } return curtoken; }
private static void GetToken() { if (TokenEnm.MoveNext()) CurrentToken = (Token)TokenEnm.Current; else CurrentToken = new Token(TokenType.END,"End"); }