public ICLS_Expression Compiler_Expression_SubValue(Token value)
 {
     if (value.type == TokenType.VALUE)
     {
         if (value.text[value.text.Length - 1] == 'f')
         {
             CLS_Value_Value<float> number = new CLS_Value_Value<float>();
             number.value_value = -float.Parse(value.text.Substring(0, value.text.Length - 1));
             return number;
         }
         else if (value.text.Contains("."))
         {
             CLS_Value_Value<double> number = new CLS_Value_Value<double>();
             number.value_value = -double.Parse(value.text);
             return number;
         }
         else
         {
             CLS_Value_Value<int> number = new CLS_Value_Value<int>();
             number.value_value = -int.Parse(value.text);
             return number;
         }
     }
     else
     {
         logger.Log_Error("无法识别的简单表达式" + value);
         return null;
     }
 }
 public ICLS_Expression Compiler_Expression_Value(Token value,int pos)
 {
     if (value.type == TokenType.VALUE)
     {
         if(value.text[value.text.Length-1]=='f')
         {
             CLS_Value_Value<float> number = new CLS_Value_Value<float>();
             number.value_value = float.Parse(value.text.Substring(0,value.text.Length-1));
             return number;
         }
         else if (value.text.Contains("."))
         {
             CLS_Value_Value<double> number = new CLS_Value_Value<double>();
             number.value_value = double.Parse(value.text);
             return number;
         }
         else
         {
             CLS_Value_Value<int> number = new CLS_Value_Value<int>();
             number.value_value = int.Parse(value.text);
             return number;
         }
     }
     else if (value.type == TokenType.STRING)
     {
         CLS_Value_Value<string> str = new CLS_Value_Value<string>();
         str.value_value = value.text.Substring(1,value.text.Length-2);
         return str;
     }
     else if (value.type == TokenType.IDENTIFIER)
     {
         CLS_Expression_GetValue getvalue = new CLS_Expression_GetValue(pos, pos, value.line, value.line);
         getvalue.value_name = value.text;
         return getvalue;
     }
     else if(value.type == TokenType.TYPE)
     {
         CLS_Expression_GetValue getvalue = new CLS_Expression_GetValue(pos, pos, value.line, value.line);
         int l = value.text.LastIndexOf('.');
         if(l>=0)
         {
             getvalue.value_name = value.text.Substring(l+1);
         }
         else
                             getvalue.value_name = value.text;
         return getvalue;
     }
     else
     {
         logger.Log_Error("无法识别的简单表达式" + value);
         return null;
     }
 }
Example #3
0
        int GetToken(string line, int nstart, out Token t)
        {
            //找到开始字符
            t.pos = nstart;
            t.line = this.line;
            t.text = " ";
            t.type = TokenType.UNKNOWN;
            if (nstart < 0) return -1;
            if (line[nstart] == '\"')
            {
                //字符串查找
                int nend = line.IndexOf('\"', nstart + 1);
                int nsub = line.IndexOf('\\', nstart + 1);
                while (nsub > 0 && nsub < nend)
                {
                    nend = line.IndexOf('\"', nsub + 2);
                    nsub = line.IndexOf('\\', nsub + 2);

                }
                if (nend - nstart + 1 < 1) throw new Exception("查找字符串失败");
                t.type = TokenType.STRING;
                int pos = nend + 1;
                t.text = line.Substring(nstart, nend - nstart + 1);
                t.text = t.text.Replace("\\\"", "\"");
                t.text = t.text.Replace("\\\'", "\'");
                t.text = t.text.Replace("\\\\", "\\");
                t.text = t.text.Replace("\\0", "\0");
                t.text = t.text.Replace("\\a", "\a");
                t.text = t.text.Replace("\\b", "\b");
                t.text = t.text.Replace("\\f", "\f");
                t.text = t.text.Replace("\\n", "\n");
                t.text = t.text.Replace("\\r", "\r");
                t.text = t.text.Replace("\\t", "\t");
                t.text = t.text.Replace("\\v", "\v");
                int sp = t.text.IndexOf('\\');
                if (sp > 0)
                {
                    throw new Exception("不可识别的转义序列:" + t.text.Substring(sp));
                }
                return pos;
            }
            else if (line[nstart] == '/')// / /= 注释
            {

                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                {
                    t.type = TokenType.PUNCTUATION;
                    t.text = line.Substring(nstart, 2);
                }
                else if (nstart < line.Length - 1 && line[nstart + 1] == '/')
                {
                    t.type = TokenType.COMMENT;
                    int enterpos = line.IndexOf('\n', nstart + 2);
                    if (enterpos < 0) t.text = line.Substring(nstart);
                    else
                        t.text = line.Substring(nstart, line.IndexOf('\n', nstart + 2) - nstart);
                }
                else
                {
                    t.type = TokenType.PUNCTUATION;
                    t.text = line.Substring(nstart, 1);
                }
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '=')//= == =>
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '>')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '!')//= ==
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '+')//+ += ++
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && (line[nstart + 1] == '=' || line[nstart + 1] == '+'))
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            //通用的一元二元运算符检查
            else if (line[nstart] == '-')//- -= -- 负数也先作为符号处理
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=' || line[nstart + 1] == '-')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '*')//* *=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '/')/// /=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '%')/// /=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '>')//> >=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '<')//< <=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }

            else if (line[nstart] == '&')//&&
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '&')
                    t.text = line.Substring(nstart, 2);
                else
                    return -1;
            }
            else if (line[nstart] == '|')//||
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '|')
                    t.text = line.Substring(nstart, 2);
                else
                    return -1;
            }
            else if (char.IsLetter(line, nstart) || line[nstart] == '_')
            {
                //字母逻辑
                //判断完整性
                int i = nstart + 1;
                while (i < line.Length && (char.IsLetterOrDigit(line, i) || line[i] == '_'))
                {
                    i++;
                }
                t.text = line.Substring(nstart, i - nstart);
                //判断字母类型: 关键字 类型 标识符
                if (keywords.Contains(t.text))
                //foreach (string s in keywords)
                {
                    //if (t.text == s)
                    {
                        t.type = TokenType.KEYWORD;
                        return nstart + t.text.Length;
                    }
                }
                if (types.Contains(t.text))
                //foreach (string s in types)
                {
                    //if (t.text == s)
                    {

                        while (line[i] == ' ' && i < line.Length)
                        {
                            i++;
                        }
                        if (line[i] == '<')/*  || line[i] == '['*/
                        {
                            int dep = 0;
                            string text = t.text;
                            while (i < line.Length)
                            {
                                if (line[i] == '<') dep++;
                                if (line[i] == '>') dep--;
                                if (line[i] == ';' || line[i] == '(' || line[i] == '{')
                                {
                                    break;
                                }
                                if (line[i] != ' ') text += line[i];
                                i++;
                                if (dep == 0)
                                {
                                    t.text = text;
                                    break;
                                }
                            }
                            //if (types.Contains(t.text))//自动注册
                            {
                                t.type = TokenType.TYPE;
                                return i;
                            }

                        }
                        else
                        {
                              t.type = TokenType.TYPE;
                            return nstart + t.text.Length;
                        }
                    }
                }
                while (line[i] == ' ' && i < line.Length)
                {
                    i++;
                }
                if (i < line.Length && (line[i] == '<'/* || line[i] == '['*/))//检查特别类型
                {
                    int dep = 0;
                    string text = t.text;
                    while (i < line.Length)
                    {
                        if (line[i] == '<')
                        {
                            dep++;
                            i++;
                            text += '<';
                            continue;
                        }
                        if (line[i] == '>')
                        {
                            dep--;
                            i++;
                            if (dep == 0)
                            {
                                t.text = text+'>';
                                break;
                            }
                            continue;
                        }
                        Token tt;
                        int nnstart = FindStart(line, i);
                        i = GetToken(line, nnstart, out tt);
                        if(tt.type!= TokenType.IDENTIFIER&&tt.type!= TokenType.TYPE&&tt.text!=",")
                        {
                            break;
                        }
                        text += tt.text;
                    }
                    if (types.Contains(t.text))
                    {
                        t.type = TokenType.TYPE;
                        return i;
                    }

                    //foreach (string s in types)
                    //{
                    //    if (s.Length > t.text.Length && line.IndexOf(s, nstart) == nstart)
                    //    {
                    //        t.type = TokenType.TYPE;
                    //        t.text = s;
                    //        return nstart + s.Length;
                    //    }

                    //}
                }
                t.type = TokenType.IDENTIFIER;
                return nstart + t.text.Length;
            }
            else if (char.IsPunctuation(line, nstart))
            {

                //else
                {
                    t.type = TokenType.PUNCTUATION;
                    t.text = line.Substring(nstart, 1);
                    return nstart + t.text.Length;
                }
                //符号逻辑
                //-号逻辑
                //"号逻辑
                ///逻辑
                //其他符号
            }
            else if (char.IsNumber(line, nstart))
            {
                //数字逻辑
                //判断数字合法性

                if (line[nstart] == '0' && line[nstart + 1] == 'x') //0x....
                {
                    int iend = nstart + 2;
                    for (int i = nstart + 2; i < line.Length; i++)
                    {
                        if (char.IsNumber(line, i))
                        {
                            iend = i;
                        }
                        else
                        {
                            break;
                        }
                    }
                    t.type = TokenType.VALUE;
                    t.text = line.Substring(nstart, iend - nstart + 1);
                }
                else
                {
                    //纯数字

                    int iend = nstart;
                    for (int i = nstart + 1; i < line.Length; i++)
                    {
                        if (char.IsNumber(line, i))
                        {
                            iend = i;
                        }
                        else
                        {
                            break;
                        }
                    }
                    t.type = TokenType.VALUE;
                    int dend = iend + 1;
                    if (dend < line.Length && line[dend] == '.')
                    {
                        int fend = dend;
                        for (int i = dend + 1; i < line.Length; i++)
                        {
                            if (char.IsNumber(line, i))
                            {
                                fend = i;
                            }
                            else
                            {
                                break;
                            }
                        }
                        if (fend + 1 < line.Length && line[fend + 1] == 'f')
                        {
                            t.text = line.Substring(nstart, fend + 2 - nstart);

                        }
                        else
                        {
                            t.text = line.Substring(nstart, fend + 1 - nstart);
                        }
                        //.111
                        //.123f
                    }
                    else
                    {
                        if (dend < line.Length && line[dend] == 'f')
                        {
                            t.text = line.Substring(nstart, dend - nstart + 1);
                        }
                        else
                        {
                            t.text = line.Substring(nstart, dend - nstart);
                        }
                    }

                }
                return nstart + t.text.Length;
            }
            else
            {
                //不可识别逻辑
                int i = nstart + 1;
                while (i < line.Length - 1 && char.IsSeparator(line, i) == false && line[i] != '\n' && line[i] != '\r' && line[i] != '\t')
                {
                    i++;
                }
                t.text = line.Substring(nstart, i - nstart);
                return nstart + t.text.Length;
            }
            //
            //    -逻辑
            //
            //    "逻辑
            //
            //    /逻辑
            //
            //    其他符号逻辑

            return nstart + t.text.Length;
        }
Example #4
0
        public IList<Token> ReadTokenList(System.IO.Stream stream)
        {
            byte[] bs = new byte[0xffff];
            stream.Read(bs, 0, 2);
            UInt16 len = BitConverter.ToUInt16(bs, 0);
            stream.Read(bs, 0, 2);
            UInt16 lenstr = BitConverter.ToUInt16(bs, 0);

            List<string> strstore = new List<string>();
            List<Token> tokens = new List<Token>();
            for (int i = 0; i < lenstr; i++)
            {
                stream.Read(bs, 0, 2);
                UInt16 slen = BitConverter.ToUInt16(bs, 0);
                stream.Read(bs, 0, slen);
                strstore.Add(System.Text.Encoding.UTF8.GetString(bs, 0, slen));
            }
            for (int i = 0; i < len; i++)
            {
                Token t = new Token();
                stream.Read(bs, 0, 4);
                UInt32 type = BitConverter.ToUInt32(bs, 0);
                t.type = (TokenType)(type % 0x0100);
                t.text = strstore[(int)(type / 0x100)];
                tokens.Add(t);
            }

            if (stream.Position < stream.Length)
            {
                UInt16 linecount = 0;
                byte[] bufu = new byte[2];
                stream.Read(bufu, 0, 2);
                linecount = BitConverter.ToUInt16(bufu, 0);
                UInt16[] linetoken = new UInt16[linecount];

                for (int i = 0; i < linecount; i++)
                {
                    stream.Read(bufu, 0, 2);
                    linetoken[i] = BitConverter.ToUInt16(bufu, 0);

                }
                int token = 0;
                for (int i = 0; i < linecount; i++)
                {
                    if ((i + 1) < linecount && linetoken[i + 1] == linetoken[i]) continue;

                    if ((i + 1) < linecount)
                    {
                        for (int j = linetoken[i]; j < linetoken[i + 1]; j++)
                        {
                            var t = tokens[j];
                            t.line = i + 1;
                            tokens[j] = t;
                        }
                    }
                    else
                    {
                        for (int j = linetoken[i]; j < tokens.Count; j++)
                        {
                            var t = tokens[j];
                            t.line = i + 1;
                            tokens[j] = t;
                        }
                    }

                    token = linetoken[i];
                }
            }
            return tokens;
        }
Example #5
0
        int GetToken(string line, int nstart, List<Token> ts, out Token t)
        {
            //找到开始字符
            t.pos = nstart;
            t.line = this.line;
            t.text = " ";
            t.type = TokenType.UNKNOWN;
            if (nstart < 0) return -1;
            if (line[nstart] == '\"')
            {
                t.text = "\"";
                int pos = nstart + 1;
                bool bend = false;
                while (pos < line.Length)
                {
                    char c = line[pos];
                    if (c == '\n')
                    {
                        throw new Exception("查找字符串失败");
                    }
                    if (c == '\"')
                    {
                        t.type = TokenType.STRING;
                        bend = true;
                        //break;
                    }
                    if (c == '\\')
                    {
                        pos++;
                        c = line[pos];
                        if (c == '\\')
                        {
                            t.text += '\\';
                            pos++;
                            continue;
                        }
                        else if (c == '"')
                        {
                            t.text += '\"';
                            pos++;
                            continue;
                        }
                        else if (c == '\'')
                        {
                            t.text += '\'';
                            pos++;
                            continue;
                        }
                        else if (c == '0')
                        {
                            t.text += '\0';
                            pos++;
                            continue;
                        }
                        else if (c == 'a')
                        {
                            t.text += '\a';
                            pos++;
                            continue;
                        }
                        else if (c == 'b')
                        {
                            t.text += '\b';
                            pos++;
                            continue;
                        }
                        else if (c == 'f')
                        {
                            t.text += '\f';
                            pos++;
                            continue;
                        }
                        else if (c == 'n')
                        {
                            t.text += '\n';
                            pos++;
                            continue;
                        }
                        else if (c == 'r')
                        {
                            t.text += '\r';
                            pos++;
                            continue;
                        }
                        else if (c == 't')
                        {
                            t.text += '\t';
                            pos++;
                            continue;
                        }
                        else if (c == 'v')
                        {
                            t.text += '\v';
                            pos++;
                            continue;
                        }
                        else
                        {
                            throw new Exception("不可识别的转义序列:" + t.text);
                        }
                    }
                    t.text += line[pos];
                    pos++;
                    if (bend)
                        return pos;
                }
                throw new Exception("查找字符串失败");
            }
            else if (line[nstart] == '\'')//char
            {
                int nend = line.IndexOf('\'', nstart + 1);
                int nsub = line.IndexOf('\\', nstart + 1);
                while (nsub > 0 && nsub < nend)
                {
                    nend = line.IndexOf('\'', nsub + 2);
                    nsub = line.IndexOf('\\', nsub + 2);

                }
                if (nend - nstart + 1 < 1) throw new Exception("查找字符失败");
                t.type = TokenType.VALUE;
                int pos = nend + 1;
                t.text = line.Substring(nstart, nend - nstart + 1);
                t.text = t.text.Replace("\\\"", "\"");
                t.text = t.text.Replace("\\\'", "\'");
                t.text = t.text.Replace("\\\\", "\\");
                t.text = t.text.Replace("\\0", "\0");
                t.text = t.text.Replace("\\a", "\a");
                t.text = t.text.Replace("\\b", "\b");
                t.text = t.text.Replace("\\f", "\f");
                t.text = t.text.Replace("\\n", "\n");
                t.text = t.text.Replace("\\r", "\r");
                t.text = t.text.Replace("\\t", "\t");
                t.text = t.text.Replace("\\v", "\v");
                int sp = t.text.IndexOf('\\');
                if (sp > 0)
                {
                    throw new Exception("不可识别的转义序列:" + t.text.Substring(sp));
                }
                if (t.text.Length > 3)
                {
                    throw new Exception("char 不可超过一个字节(" + t.line + ")");
                }
                return pos;
            }
            else if (line[nstart] == '#')   // #开头的都当作注释,如#region #endregion
            {
                t.type = TokenType.COMMENT;
                int enterpos = line.IndexOf('\n', nstart + 2);
                if (enterpos < 0) t.text = line.Substring(nstart);
                else
                    t.text = line.Substring(nstart, enterpos - nstart);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '/')   // / /= 注释
            {
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                {
                    t.type = TokenType.PUNCTUATION;
                    t.text = line.Substring(nstart, 2);
                }
                else if (nstart < line.Length - 1 && line[nstart + 1] == '/')
                {
                    t.type = TokenType.COMMENT;
                    int enterpos = line.IndexOf('\n', nstart + 2);
                    if (enterpos < 0) t.text = line.Substring(nstart);
                    else
                        t.text = line.Substring(nstart, enterpos - nstart);
                }
                else if (nstart < line.Length - 1 && line[nstart + 1] == '*')
                {
                    t.type = TokenType.COMMENT;
                    int enterpos = line.IndexOf("*/", nstart + 2);
                    t.text = line.Substring(nstart, enterpos + 2 - nstart);
                }
                else
                {
                    t.type = TokenType.PUNCTUATION;
                    t.text = line.Substring(nstart, 1);
                }
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '=')//= == =>
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '>')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '!')//= ==
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '+')//+ += ++
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && (line[nstart + 1] == '=' || line[nstart + 1] == '+'))
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            //通用的一元二元运算符检查
            else if (line[nstart] == '-')//- -= -- 负数也先作为符号处理
            {
                // 先检测是否是负数
                if (ts.Count > 0)
                {
                    Token preToken = ts[ts.Count - 1];
                    if (preToken.type == TokenType.PUNCTUATION &&
                        preToken.text != ")" &&
                        preToken.text != "]")
                    {
                        bool isNegativeValue = false;
                        bool isHexNum = false;
                        int negativeValueEnd = nstart + 1;
                        for (; negativeValueEnd < line.Length; negativeValueEnd++)
                        {
                            char c = line[negativeValueEnd];

                            if (char.IsNumber(c))
                            {
                                isNegativeValue = true;
                                continue;
                            }
                            if (c == '.')
                            {
                                isNegativeValue = true;
                                continue;
                            }
                            if (isNegativeValue && c == 'f')
                            {
                                continue;
                            }
                            if (isNegativeValue && (negativeValueEnd == nstart + 2) && (c == 'x' || c == 'X'))
                            {
                                isHexNum = true;
                                continue;
                            }
                            if (isHexNum && (c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'))
                            {
                                continue;
                            }

                            break;
                        }
                        if (isNegativeValue)
                        {
                            t.type = TokenType.VALUE;
                            t.text = line.Substring(nstart, negativeValueEnd - nstart);
                            return nstart + t.text.Length;
                        }
                    }
                }
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=' || line[nstart + 1] == '-')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '*')//* *=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '/')/// /=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '%')/// /=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '>')//> >= >> >>=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '>')
                {
                    if (nstart < line.Length - 2 && line[nstart + 2] == '=')
                        t.text = line.Substring(nstart, 3);
                    else
                        t.text = line.Substring(nstart, 2);
                }
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '<')//< <= << <<=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '<')
                {
                    if (nstart < line.Length - 2 && line[nstart + 2] == '=')
                        t.text = line.Substring(nstart, 3);
                    else
                        t.text = line.Substring(nstart, 2);
                }
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '&')//& && &=
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '&')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '|')//| ||
            {
                t.type = TokenType.PUNCTUATION;
                if (nstart < line.Length - 1 && line[nstart + 1] == '|')
                    t.text = line.Substring(nstart, 2);
                else if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '~')//~
            {
                t.type = TokenType.PUNCTUATION;
                t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '^')//^ ^=
            {
                if (nstart < line.Length - 1 && line[nstart + 1] == '=')
                    t.text = line.Substring(nstart, 2);
                else
                    t.text = line.Substring(nstart, 1);
                t.type = TokenType.PUNCTUATION;
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '[' && line.Substring(nstart, line.IndexOf("]", nstart) + 1 - nstart) == "[ProtoContract]")        // proto头判断
            {
                t.text = line.Substring(nstart, line.IndexOf("]", nstart) + 1 - nstart);
                t.type = TokenType.COMMENT;
                return nstart + t.text.Length;
            }
            else if (line[nstart] == '[' && line.Substring(nstart, line.IndexOf("]", nstart) + 1 - nstart).StartsWith("[ProtoMember"))  // proto字段判断
            {
                string tempText = line.Substring(nstart, line.IndexOf("]", nstart) + 1 - nstart);
                int pos1 = tempText.IndexOf("(");
                int pos2 = tempText.IndexOf(")");
                t.text = tempText.Substring(pos1 + 1, pos2 - pos1 - 1);
                t.type = TokenType.ProtoIndex;
                return nstart + tempText.Length;
            }
            else if (char.IsLetter(line, nstart) || line[nstart] == '_')
            {
                //字母逻辑
                //判断完整性
                int i = nstart + 1;
                while (i < line.Length && (char.IsLetterOrDigit(line, i) || line[i] == '_'))
                {
                    i++;
                }
                t.text = line.Substring(nstart, i - nstart);
                //判断字母类型: 关键字 类型 标识符
                if (keywords.Contains(t.text))
                {
                    // 将 using namespace;当作注释
                    if (t.text == "using")
                    {
                        t.type = TokenType.COMMENT;
                        return line.IndexOf(';', nstart) + 1;
                    }

                    t.type = TokenType.KEYWORD;
                    return nstart + t.text.Length;
                }
                if (dictTypeKeywords.ContainsKey(t.text))
                {
                    while (line[i] == ' ' && i < line.Length)
                    {
                        i++;
                    }
                    if (line[i] == '<')/*  || line[i] == '['*/
                    {
                        int dep = 0;
                        string text = t.text;
                        while (i < line.Length)
                        {
                            if (line[i] == '<') dep++;
                            if (line[i] == '>') dep--;
                            if (line[i] == ';' || line[i] == '(' || line[i] == '{')
                            {
                                break;
                            }
                            if (line[i] != ' ') text += line[i];
                            i++;
                            if (dep == 0)
                            {
                                t.text = text;
                                break;
                            }
                        }
                        t.type = TokenType.TYPE;
                        return i;
                    }
                    else
                    {
                        t.type = TokenType.TYPE;
                        return nstart + t.text.Length;
                    }
                }
                while (i < line.Length && line[i] == ' ')
                {
                    i++;
                }
                if (i < line.Length && (line[i] == '<'/* || line[i] == '['*/))//检查特别类型
                {
                    int dep = 0;
                    string text = t.text;
                    while (i < line.Length)
                    {
                        if (line[i] == '<')
                        {
                            dep++;
                            i++;
                            text += '<';
                            continue;
                        }
                        if (line[i] == '>')
                        {
                            dep--;
                            i++;
                            if (dep == 0)
                            {
                                t.text = text + '>';
                                break;
                            }
                            continue;
                        }
                        Token tt;
                        int nnstart = FindStart(line, i);
                        i = GetToken(line, nnstart, ts, out tt);
                        if (tt.type != TokenType.IDENTIFIER && tt.type != TokenType.TYPE && tt.text != ",")
                        {
                            break;
                        }
                        text += tt.text;
                    }
                    if (dictTypeKeywords.ContainsKey(t.text))
                    {
                        t.type = TokenType.TYPE;
                        return i;

                    }
                    else if (dep == 0)
                    {
                        t.type = TokenType.IDENTIFIER;
                        return i;
                    }
                }
                t.type = TokenType.IDENTIFIER;
                return nstart + t.text.Length;
            }
            else if (char.IsPunctuation(line, nstart))
            {
                t.type = TokenType.PUNCTUATION;
                t.text = line.Substring(nstart, 1);
                return nstart + t.text.Length;
            }
            else if (char.IsNumber(line, nstart))
            {
                // 16进制判断
                if (line[nstart] == '0' && (line[nstart + 1] == 'x' || line[nstart + 1] == 'X'))
                {
                    int iend = nstart + 2;
                    for (int i = nstart + 2; i < line.Length; i++)
                    {
                        char c = line[i];
                        if (char.IsNumber(c) || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')
                        {
                            iend = i;
                        }
                        else
                        {
                            break;
                        }
                    }
                    t.type = TokenType.VALUE;
                    t.text = line.Substring(nstart, iend - nstart + 1);
                }
                else
                {
                    //纯数字

                    int iend = nstart;
                    for (int i = nstart + 1; i < line.Length; i++)
                    {
                        if (char.IsNumber(line, i))
                        {
                            iend = i;
                        }
                        else
                        {
                            break;
                        }
                    }
                    t.type = TokenType.VALUE;
                    int dend = iend + 1;
                    if (dend < line.Length && line[dend] == '.')
                    {
                        int fend = dend;
                        for (int i = dend + 1; i < line.Length; i++)
                        {
                            if (char.IsNumber(line, i))
                            {
                                fend = i;
                            }
                            else
                            {
                                break;
                            }
                        }
                        if (fend + 1 < line.Length && line[fend + 1] == 'f')
                        {
                            t.text = line.Substring(nstart, fend + 2 - nstart);
                        }
                        else
                        {
                            t.text = line.Substring(nstart, fend + 1 - nstart);
                        }
                    }
                    else
                    {
                        if (dend < line.Length && line[dend] == 'f')
                        {
                            t.text = line.Substring(nstart, dend - nstart + 1);
                        }
                        else
                        {
                            t.text = line.Substring(nstart, dend - nstart);
                        }
                    }
                }
                return nstart + t.text.Length;
            }
            else
            {
                //不可识别逻辑
                int i = nstart + 1;
                while (i < line.Length - 1 && char.IsSeparator(line, i) == false && line[i] != '\n' && line[i] != '\r' && line[i] != '\t')
                {
                    i++;
                }
                t.text = line.Substring(nstart, i - nstart);
                return nstart + t.text.Length;
            }
        }
Example #6
0
        public IList<Token> ReadTokenList(System.IO.Stream stream)
        {
            byte[] bs = new byte[0xffff];
            // 读取token数量
            stream.Read(bs, 0, 2);
            UInt16 len = BitConverter.ToUInt16(bs, 0);
            // 读取字符串数量
            stream.Read(bs, 0, 2);
            UInt16 lenstr = BitConverter.ToUInt16(bs, 0);

            // 读取字符串
            List<string> listText = new List<string>();
            for (int i = 0; i < lenstr; i++)
            {
                int bsLen;
                ReadUInt16Variant(stream, out bsLen);
                stream.Read(bs, 0, bsLen);
                listText.Add(System.Text.Encoding.UTF8.GetString(bs, 0, bsLen));
            }

            // 创建token
            List<Token> tokens = new List<Token>();
            if (listText.Count > 0xff)
            {
                for (int i = 0; i < len; i++)
                {
                    stream.Read(bs, 0, 3);
                    Token t = new Token();
                    t.type = (TokenType)bs[0];
                    t.text = listText[BitConverter.ToUInt16(bs, 1)];
                    tokens.Add(t);
                }
            }
            else
            {
                for (int i = 0; i < len; i++)
                {
                    stream.Read(bs, 0, 2);
                    Token t = new Token();
                    t.type = (TokenType)bs[0];
                    t.text = listText[bs[1]];
                    tokens.Add(t);
                }
            }

            return tokens;
        }
 public ICLS_Expression Compiler_Expression_Value(Token value, int pos)
 {
     if (value.type == TokenType.VALUE)
     {
         if (value.text.StartsWith("-0x") || value.text.StartsWith("-0X"))
         {
             long lv = -Convert.ToInt64(value.text.Substring(1), 16);
             return CreateValueExpression(lv);
         }
         else if (value.text.StartsWith("0x") || value.text.StartsWith("0X"))
         {
             long lv = Convert.ToInt64(value.text, 16);
             return CreateValueExpression(lv);
         }
         else if (value.text[value.text.Length - 1] == 'f')
         {
             CLS_Expression_Value<float> number = new CLS_Expression_Value<float>();
             number.value = float.Parse(value.text.Substring(0, value.text.Length - 1));
             return number;
         }
         else if (value.text.Contains("."))
         {
             CLS_Expression_Value<double> number = new CLS_Expression_Value<double>();
             number.value = double.Parse(value.text);
             return number;
         }
         else if (value.text.Contains("'"))
         {
             CLS_Expression_Value<char> number = new CLS_Expression_Value<char>();
             number.value = (char)value.text[1];
             return number;
         }
         else if (value.text.StartsWith("-"))
         {
             long lv = long.Parse(value.text);
             return CreateValueExpression(lv);
         }
         else
         {
             long lv = long.Parse(value.text);
             return CreateValueExpression(lv);
         }
     }
     else if (value.type == TokenType.STRING)
     {
         CLS_Expression_Value<string> str = new CLS_Expression_Value<string>();
         str.value = value.text.Substring(1, value.text.Length - 2);
         return str;
     }
     else if (value.type == TokenType.IDENTIFIER)
     {
         CLS_Expression_GetValue getvalue = new CLS_Expression_GetValue(pos, pos, value.line, value.line);
         getvalue.value_name = value.text;
         return getvalue;
     }
     else if (value.type == TokenType.TYPE)
     {
         CLS_Expression_GetValue getvalue = new CLS_Expression_GetValue(pos, pos, value.line, value.line);
         int l = value.text.LastIndexOf('.');
         if (l >= 0)
         {
             getvalue.value_name = value.text.Substring(l + 1);
         }
         else
             getvalue.value_name = value.text;
         return getvalue;
     }
     logger.Log_Error("无法识别的简单表达式" + value);
     return null;
 }
 public ICLS_Expression Compiler_Expression_SubValue(Token value)
 {
     logger.Log_Error("已经整合到Compiler_Expression_Value处理" + value);
     return null;
 }