Beispiel #1
0
 public void Add(LexToken tok)
 {
     if (Tokens.Count == 0)
     {
         StartCol = tok.Col;
     }
     Tokens.Add(tok);
 }
Beispiel #2
0
 private void addIdentOrKey(LexToken token)
 {
     //if (processKeyIdent(token, "如果",TokenKindSymbol.IF)) return;
     //if (processKeyIdent(token, "否则如果", TokenKindSymbol.ELSEIF)) return;
     //if (processKeyIdent(token, "否则", TokenKindSymbol.ELSE)) return;
     //if (processKeyIdent(token, "循环当", TokenKind.While)) return;
     //if (processKeyIdent(token, "重复", TokenKindSymbol.Repeat)) return;
     //if (processKeyIdent(token, "当", TokenKindSymbol.Dang)) return;
     //if (processKeyIdent(token, "次", TokenKind.Times)) return;
     //if (processKeyIdent(token, "处理", TokenKindSymbol.Catch)) return;
     //if (processKeyIdent(token, "循环每一个", TokenKindSymbol.Foreach)) return;
     curToks.Add(token);
 }
Beispiel #3
0
        public List <LineTokenCollection> Scan(SourceReader sr, ContextFile fileContext, int startLine)
        {
            Reset(sr, fileContext);
            line    = startLine;
            curToks = new LineTokenCollection();
            //report("开始");
            //InitLineFirst();

            while (ch != END)
            {
                //report("ch="+ch+" "+(int)ch);
                //if (line == 33  )// ch == '控' && line == 18)
                //{
                //    //report("col:" + col);
                //    Widther.IsDebug = true;
                //}
                //else if (  line == 34)// ch == '控' && line == 18)
                //{
                //    Widther.IsDebug = true;
                //}
                //else
                //{
                //    Widther.IsDebug = false;
                //}

                char nextChar = GetNext();
                if (ch == ' ' || ch == '\t')
                {
                    //report("SkipSpace");
                    SkipWhiteSpace();
                }
                else if (ch == '/' && nextChar == '/')
                {
                    SkipSingleLineComment();
                }
                else if (ch == '/' && nextChar == '*')
                {
                    SkipMutilLineComment();
                }
                else if (ch == '/')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.DIV);// { Col = col, Line = line, };
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == '"' || ch == '“' || ch == '”')
                {
                    string          str = scanString();
                    LexTokenLiteral tok = new LexTokenLiteral(line, col - 1, TokenKindLiteral.LiteralString, str);// { Col = col - 1, Line = line, Text = str, Kind = TokenKindSymbol.LiteralString };
                    curToks.Add(tok);
                }
                else if (ch == '\r' && nextChar == '\n')
                {
                    Next(); Next();
                    AddLineToken();//lineTokens.Add(curToks);
                    curToks = new LineTokenCollection();
                    ScanNewLine();
                }
                else if (ch == '\n' || ch == '\r')
                {
                    Next();
                    AddLineToken(); //lineTokens.Add(curToks);
                    curToks = new LineTokenCollection();
                    ScanNewLine();
                }
                else if ("0123456789".IndexOf(ch) != -1)
                {
                    string str  = scanNumber();
                    var    temp = col;
                    if (StringHelper.IsInt(str))
                    {
                        LexTokenLiteral tok = new LexTokenLiteral(line, temp, TokenKindLiteral.LiteralInt, str);
                        curToks.Add(tok);
                    }
                    else if (StringHelper.IsFloat(str))
                    {
                        LexTokenLiteral tok = new LexTokenLiteral(line, temp, TokenKindLiteral.LiteralFloat, str);
                        curToks.Add(tok);
                    }
                    else
                    {
                        lexError(str + "不是正确的数字");
                    }
                }
                else if (ch == '+')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.ADD);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == '-')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.SUB);
                    curToks.Add(tok);
                    Next();
                }
                else if ((ch == '=') && (nextChar == '='))
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.EQ);
                    curToks.Add(tok);
                    Next(); Next();
                }
                else if ((ch == '=') && (nextChar == '>'))
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.AssignTo);
                    curToks.Add(tok);
                    Next(); Next();
                }
                else if ((ch == '='))
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.Assign);
                    curToks.Add(tok);
                    Next();
                }
                else if ((ch == '*'))
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.MUL);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == ',' || ch == ',')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.Comma);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == ';' || ch == ';')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.Semi);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == '(' || ch == '(')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.LBS);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == ')' || ch == ')')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.RBS);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == '>' && GetNext() == '=')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.GE);
                    curToks.Add(tok);
                    Next(); Next();
                }
                else if (ch == '>')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.GT);
                    curToks.Add(tok);
                    Next();
                }
                else if (ch == '<' && nextChar == '=')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.LE);
                    curToks.Add(tok);
                    Next(); Next();
                }
                else if (ch == '<')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.LT);
                    curToks.Add(tok);
                    Next();
                }
                else if ((nextChar == '!' || nextChar == '!') && (nextChar == '=' || nextChar == '='))
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.NE);
                    curToks.Add(tok);
                    Next(); Next();
                }
                else if (ch == ':' || ch == ':')
                {
                    LexTokenSymbol tok = new LexTokenSymbol(line, col, TokenKindSymbol.Colon);// { Col = col, Line = line, Kind = TokenKindSymbol.Colon };
                    curToks.Add(tok);
                    Next();
                }
                else if ((ch >= 'A' && ch <= 'Z') /*|| (ch == '_') */ || (ch >= 'a' && ch <= 'z') || ChineseHelper.IsChineseLetter(ch))
                {
                    var      tempCol  = col;
                    var      tempLine = line;
                    LexToken t1       = scanIdentToken();
                    if (t1.Text == "说明")
                    {
                        if (ch == ':' || ch == ':')
                        {
                            SkipSingleLineComment();
                            continue;;
                        }
                    }
                    addIdentOrKey(t1);
                }
                else if (char.IsControl(ch))
                {
                    while (char.IsControl(ch) && ch != END)
                    {
                        Next();
                        if ((int)ch == 13)
                        {
                            ScanNewLine();
                        }
                    }
                }
                else
                {
                    lexError("无法识别" + (int)ch + ": '" + ch + "' ");
                    Next();
                }
            }
            if (curToks != null && curToks.Count > 0)
            {
                AddLineToken();
            }
            return(lineTokens);
        }