コード例 #1
0
ファイル: Lexer.cs プロジェクト: ronsaldo/chela
        private bool CheckGenericStart()
        {
            // Store the old state.
            int oldLine = line;
            int oldColumn = column;
            int oldLook = look;
            long oldPosition = stream.Position;

            // Start with GENERIC_START.
            int token = Token.GENERIC_START;

            // Make sure there are only comments, ',', '.', '*', '<' and identifiers
            // before hitting '>'.
            int genericCheckCount = 1;
            bool foundDot = false;
            for(;;)
            {
                // Preprocess
                int firstLook = Preprocess();
                if(firstLook == 0)
                    firstLook = look;

                // If hit EOF, end loop.
                if(firstLook == EOF)
                {
                    token = '<';
                    break;
                }

                if(genericCheckCount == 0)
                {
                    foundDot = firstLook == '.';
                    break;
                }

                // Check for identifiers.
                if(IsAlpha(firstLook) || firstLook == '_')
                {
                    // Identifier.
                    Next();
                    while(IsAlpha(look) || IsDigit(look) || look == '_')
                        Next();

                    // Consume the next token.
                    continue;
                }

                // Check for end single character tokens.
                if(firstLook == ',' || firstLook == '.' || firstLook == '*')
                {
                    // Consume the token.
                    Next();
                }
                else if(firstLook == '<')
                {
                    // Consume the token.
                    Next();
                    ++genericCheckCount;

                }
                else if(firstLook == '>')
                {
                    // Consume the token.
                    Next();
                    --genericCheckCount;
                    if(genericCheckCount == 0)
                        continue;
                }
                else
                {
                    // This is not a generic.
                    token = '<';
                    break;
                }

            }

            // Restore the old state.
            line = oldLine;
            column = oldColumn;
            look = oldLook;
            stream.Seek(oldPosition, SeekOrigin.Begin);

            // Increase the generic count.
            if(token == Token.GENERIC_START)
            {
                if(foundDot)
                {
                    token = Token.GENERIC_START_DOT;
                    genericStack.Add(Token.GENERIC_END_DOT);
                }
                else
                {
                    genericStack.Add(Token.GENERIC_END);
                }
            }

            // Return the token.
            tokenValue = new TokenValue(token, fileName, line, column);
            return true;
        }
コード例 #2
0
ファイル: Lexer.cs プロジェクト: ronsaldo/chela
        public Lexer(string fileName, Stream stream)
        {
            this.fileName = fileName;
            this.stream = stream;
            this.line = 1;
            this.column = 0;
            this.tokenValue = null;
            this.keywords = new Dictionary<string, int> ();
            this.definitions = new Dictionary<string, bool> ();
            this.preprocessorStates = new List<bool> ();
            this.ignoring = false;
            this.ignoreCount = 0;
            this.hasBeenTruePrep = false;
            this.start = true;
            this.genericStack = new List<int> ();

            // Look the first character.
            Next();

            // Set the keywords.
            this.keywords.Add("bool", Token.KBOOL);
            this.keywords.Add("byte", Token.KBYTE);
            this.keywords.Add("sbyte", Token.KSBYTE);
            this.keywords.Add("char", Token.KCHAR);
            this.keywords.Add("uchar", Token.KUCHAR);
            this.keywords.Add("double", Token.KDOUBLE);
            this.keywords.Add("float", Token.KFLOAT);
            this.keywords.Add("int", Token.KINT);
            this.keywords.Add("uint", Token.KUINT);
            this.keywords.Add("long", Token.KLONG);
            this.keywords.Add("ulong", Token.KULONG);
            this.keywords.Add("short", Token.KSHORT);
            this.keywords.Add("ushort", Token.KUSHORT);
            this.keywords.Add("object", Token.KOBJECT);
            this.keywords.Add("string", Token.KSTRING);
            this.keywords.Add("size_t", Token.KSIZE_T);
            this.keywords.Add("ptrdiff_t", Token.KPTRDIFF_T);
            this.keywords.Add("void", Token.KVOID);
            this.keywords.Add("vec2", Token.KVEC2);
            this.keywords.Add("vec3", Token.KVEC3);
            this.keywords.Add("vec4", Token.KVEC4);
            this.keywords.Add("dvec2", Token.KDVEC2);
            this.keywords.Add("dvec3", Token.KDVEC3);
            this.keywords.Add("dvec4", Token.KDVEC4);
            this.keywords.Add("ivec2", Token.KIVEC2);
            this.keywords.Add("ivec3", Token.KIVEC3);
            this.keywords.Add("ivec4", Token.KIVEC4);
            this.keywords.Add("bvec2", Token.KBVEC2);
            this.keywords.Add("bvec3", Token.KBVEC3);
            this.keywords.Add("bvec4", Token.KBVEC4);

            this.keywords.Add("mat2"  , Token.KMAT2);
            this.keywords.Add("mat2x3", Token.KMAT2x3);
            this.keywords.Add("mat2x4", Token.KMAT2x4);
            this.keywords.Add("mat3x2", Token.KMAT3x2);
            this.keywords.Add("mat3"  , Token.KMAT3);
            this.keywords.Add("mat3x4", Token.KMAT3x4);
            this.keywords.Add("mat4x2", Token.KMAT4x2);
            this.keywords.Add("mat4x3", Token.KMAT4x3);
            this.keywords.Add("mat4"  , Token.KMAT4);

            this.keywords.Add("dmat2"  , Token.KDMAT2);
            this.keywords.Add("dmat2x3", Token.KDMAT2x3);
            this.keywords.Add("dmat2x4", Token.KDMAT2x4);
            this.keywords.Add("dmat3x2", Token.KDMAT3x2);
            this.keywords.Add("dmat3"  , Token.KDMAT3);
            this.keywords.Add("dmat3x4", Token.KDMAT3x4);
            this.keywords.Add("dmat4x2", Token.KDMAT4x2);
            this.keywords.Add("dmat4x3", Token.KDMAT4x3);
            this.keywords.Add("dmat4"  , Token.KDMAT4);

            this.keywords.Add("imat2"  , Token.KIMAT2);
            this.keywords.Add("imat2x3", Token.KIMAT2x3);
            this.keywords.Add("imat2x4", Token.KIMAT2x4);
            this.keywords.Add("imat3x2", Token.KIMAT3x2);
            this.keywords.Add("imat3"  , Token.KIMAT3);
            this.keywords.Add("imat3x4", Token.KIMAT3x4);
            this.keywords.Add("imat4x2", Token.KIMAT4x2);
            this.keywords.Add("imat4x3", Token.KIMAT4x3);
            this.keywords.Add("imat4"  , Token.KIMAT4);

            this.keywords.Add("typedef", Token.KTYPEDEF);
            this.keywords.Add("class", Token.KCLASS);
            this.keywords.Add("delegate", Token.KDELEGATE);
            this.keywords.Add("event", Token.KEVENT);
            this.keywords.Add("interface", Token.KINTERFACE);
            this.keywords.Add("enum", Token.KENUM);
            this.keywords.Add("namespace", Token.KNAMESPACE);
            this.keywords.Add("struct", Token.KSTRUCT);
            this.keywords.Add("operator", Token.KOPERATOR);

            this.keywords.Add("public", Token.KPUBLIC);
            this.keywords.Add("internal", Token.KINTERNAL);
            this.keywords.Add("protected", Token.KPROTECTED);
            this.keywords.Add("private", Token.KPRIVATE);
            this.keywords.Add("extern", Token.KEXTERN);
            this.keywords.Add("kernel", Token.KKERNEL);
            this.keywords.Add("virtual", Token.KVIRTUAL);
            this.keywords.Add("static", Token.KSTATIC);
            this.keywords.Add("override", Token.KOVERRIDE);
            this.keywords.Add("abstract", Token.KABSTRACT);
            this.keywords.Add("sealed", Token.KSEALED);
            this.keywords.Add("const", Token.KCONST);
            this.keywords.Add("readonly", Token.KREADONLY);
            this.keywords.Add("unsafe", Token.KUNSAFE);
            this.keywords.Add("partial", Token.KPARTIAL);

            this.keywords.Add("__cdecl", Token.KCDECL);
            this.keywords.Add("__stdcall", Token.KSTDCALL);
            this.keywords.Add("__apicall", Token.KAPICALL);

            this.keywords.Add("break", Token.KBREAK);
            this.keywords.Add("case", Token.KCASE);
            this.keywords.Add("catch", Token.KCATCH);
            this.keywords.Add("continue", Token.KCONTINUE);
            this.keywords.Add("default", Token.KDEFAULT);
            this.keywords.Add("do", Token.KDO);
            this.keywords.Add("else", Token.KELSE);
            this.keywords.Add("finally", Token.KFINALLY);
            this.keywords.Add("fixed", Token.KFIXED);
            this.keywords.Add("for", Token.KFOR);
            this.keywords.Add("foreach", Token.KFOREACH);
            this.keywords.Add("goto", Token.KGOTO);
            this.keywords.Add("if", Token.KIF);
            this.keywords.Add("in", Token.KIN);
            this.keywords.Add("lock", Token.KLOCK);
            this.keywords.Add("return", Token.KRETURN);
            this.keywords.Add("while", Token.KWHILE);
            this.keywords.Add("using", Token.KUSING);
            this.keywords.Add("switch", Token.KSWITCH);
            this.keywords.Add("throw", Token.KTHROW);
            this.keywords.Add("try", Token.KTRY);

            this.keywords.Add("null", Token.KNULL);
            this.keywords.Add("this", Token.KTHIS);
            this.keywords.Add("base", Token.KBASE);

            this.keywords.Add("bitand", Token.BITAND);
            this.keywords.Add("bitor", Token.BITOR);
            this.keywords.Add("compl", Token.BITNOT);
            this.keywords.Add("xor", Token.BITXOR);
            this.keywords.Add("and", Token.LAND);
            this.keywords.Add("or", Token.LOR);
            this.keywords.Add("not", Token.LNOT);
            this.keywords.Add("sizeof", Token.SIZEOF);
            this.keywords.Add("typeof", Token.TYPEOF);
            this.keywords.Add("is", Token.IS);
            this.keywords.Add("as", Token.AS);
            this.keywords.Add("new", Token.NEW);
            this.keywords.Add("heapalloc", Token.HEAPALLOC);
            this.keywords.Add("stackalloc", Token.STACKALLOC);
            this.keywords.Add("delete", Token.KDELETE);
            this.keywords.Add("reinterpret_cast", Token.REINTERPRET_CAST);
            this.keywords.Add("checked", Token.CHECKED);
            this.keywords.Add("unchecked", Token.UNCHECKED);

            this.keywords.Add("ref", Token.KREF);
            this.keywords.Add("out", Token.KOUT);
            this.keywords.Add("params", Token.KPARAMS);
        }
コード例 #3
0
ファイル: Lexer.cs プロジェクト: ronsaldo/chela
        /** move on to next token.
            @return false if positioned beyond tokens.
            @throws IOException on input error.
          */
        public virtual bool advance()
        {
            // Perform preprocessing.
            int c1 = Preprocess();

            // Store the line and column.
            int line = this.line;
            int column = this.column;

            // Read numbers.
            bool afterPeriod = false;
            if(c1 == 0 && look == '.')
            {
                afterPeriod = true;
                Next();
                if(!IsDigit(look))
                {
                    // It was only a period.
                    tokenValue = new TokenValue('.', fileName, line, column);
                    return true;
                }
            }

            // Check for numbers of the form ([0-9].[0-9]*(e[+-]?[0-9]+)?) | .[0-9]+(e[+-]?[0-9]+)?
            if(c1 == 0 && IsDigit(look))
            {
                ulong mantissa = 0;
                int exponent = 0;

                // Check for octal and hex.
                bool otherBase = false;
                if(look == '0' && !afterPeriod)
                {
                    Next();
                    if(look == 'x' || look == 'X')
                    {
                        // Set the other base flag.
                        otherBase = true;

                        // Hexadecimal number.
                        Next();
                        while(IsDigit(look) ||
                              (look >= 'a' && look <= 'f') ||
                              (look >= 'A' && look <= 'F'))
                        {
                            // Increase the mantissa.
                            mantissa *= 0x10;

                            // Append the digit.
                            if(look >= '0' && look <= '9')
                            {
                                mantissa += (ulong)(look - '0');
                            }
                            else if(look >= 'a' && look <= 'f')
                            {
                                mantissa += (ulong)(look - 'a' + 0xA);
                            }
                            else if(look >= 'A' && look <= 'F')
                            {
                                mantissa += (ulong)(look - 'A' + 0xA);
                            }

                            // Read the next digit.
                            Next();
                        }
                    }
                    else
                    {
                        // Set the other base flag.
                        otherBase = true;

                        // Octal number.
                        while(look >= '0' && look <= '7')
                        {
                            // Increase the mantissa.
                            mantissa *= 010u;

                            // Append the digit.
                            if(look >= '0' && look <= '7')
                                mantissa += (ulong)(look - '0');

                            // Read the next digit.
                            Next();
                        }

                        if(IsDigit(look) || look == '.' || look == 'e' || look == 'E')
                        {
                            // Unset the other base.
                            otherBase = false;

                            // Convert the octal into decimal.
                            mantissa = ulong.Parse(System.Convert.ToString((long)mantissa, 8));
                        }
                    }

                }

                if(!otherBase)
                {
                    while(IsDigit(look) || look == '.' || look == 'e' || look == 'E')
                    {
                        if(look == '.')
                        {
                            if(afterPeriod)
                                break;
                            afterPeriod = true;
                        }
                        else if(look == 'e' || look == 'E')
                        {
                            // Set the after period flag, to make sure we output a floating point number.
                            afterPeriod = true;

                            // Read the exponent sign.
                            Next();
                            bool pos = true;
                            if(look == '+')
                            {
                                Next();
                            }
                            else if(look == '-')
                            {
                                pos = false;
                                Next();
                            }

                            // Read the exponent.
                            int extraExponent = 0;
                            while(IsDigit(look))
                            {
                                extraExponent = extraExponent*10 + look - '0';
                                Next();
                            }

                            // Update the full exponent.
                            exponent += pos ? extraExponent : -extraExponent;

                            // Break of the loop.
                            break;
                        }
                        else
                        {
                            mantissa = mantissa*10 + (ulong)(look - '0');
                            if(afterPeriod)
                                exponent -= 1;
                        }

                        // Read the next character.
                        Next();
                    }
                }

                // Read the suffix.
                int suffix = 0;
                if(IsAlpha(look))
                {
                    suffix = look;
                    Next();
                }

                // Create the token.
                if(afterPeriod)
                {
                    // Check the suffix.
                    int tok = Token.DOUBLE;
                    if(suffix == 'f' || suffix == 'F')
                        tok = Token.FLOAT;
                    else if(suffix != 'd' && suffix != 'D' && suffix != 0)
                    {
                        // Invalid suffix.
                        Error("Invalid floating point suffix '" + (char)suffix + "'");
                    }

                    // Make the value.
                    double value = (double)mantissa;
                    int exp = exponent;
                    if(exp < 0) exp = -exp;
                    for(int i = 0; i < exp; i++)
                    {
                        if(exponent > 0)
                            value *= 10.0;
                        else
                            value *= 0.1;
                    }

                    // Create the token.
                    tokenValue = new FloatingPointToken(value, tok, fileName, line, column);
                }
                else
                {
                    if(suffix == 'u' || suffix == 'U')
                    {
                        if(look == 'l' || look == 'L')
                        {
                            // ulong constant.
                            Next();

                            // Create the token.
                            tokenValue = new UIntegralToken(mantissa, Token.ULONG, fileName, line, column);
                        }
                        else
                        {
                            // Create the token.
                            tokenValue = new UIntegralToken(mantissa, Token.UINTEGER, fileName, line, column);
                        }
                    }
                    else
                    {
                        if(suffix == 'l' || suffix == 'L')
                        {
                            // Create the token.
                            tokenValue = new IntegralToken((long)mantissa, Token.LONG, fileName, line, column);
                        }
                        else // Create the token, using the least capable type.
                        {
                            int tok = Token.INTEGER;
                            if(mantissa > 0x7FFFFFFF)
                                tok = Token.LONG;
                            tokenValue = new IntegralToken((long)mantissa, tok, fileName, line, column);
                        }
                    }
                }

                return true;
            }

            // Check for C-string.
            bool cstring = false;
            string identprefix = null;
            if(c1 == 0 && look == 'c')
            {
                Next();
                if(look == '"')
                    cstring = true;
                else
                    identprefix = "c";
            }

            // Read identifiers.
            if(c1 == 0 &&  (IsAlpha(look) || look == '_' || identprefix != null))
            {
                // Build the identifier.
                string identifier = ReadIdentifier(identprefix);

                // Boolean values has special treatment.
                if(identifier == "true")
                {
                    tokenValue = new BoolToken(true, Token.BOOL, fileName, line, column);
                    return true;
                }
                else if(identifier == "false")
                {
                    tokenValue = new BoolToken(false, Token.BOOL, fileName, line, column);
                    return true;
                }

                // Check for keywords.
                int key;
                if(!keywords.TryGetValue(identifier, out key))
                    key = Token.IDENTIFIER;

                // Create the token.
                tokenValue = new WordToken(identifier, key,
                                           fileName, line, column);
                return true;
            }

            // Read raw identifiers.
            if(c1 == 0 && look == '@')
            {
                // Skip the @.
                Next();

                // Read the identifier.
                string identifier = ReadIdentifier();

                // Create the token.
                tokenValue = new WordToken(identifier, Token.IDENTIFIER,
                                           fileName, line, column);
                return true;
            }

            // Read literals.
            if(c1 == 0 && look == '"')
            {
                // Create the token.
                tokenValue = new WordToken(ReadLiteral(), cstring ? Token.CSTRING : Token.STRING, fileName, line, column);
                return true;
            }

            // Read characters.
            if(c1 == 0 && look == '\'')
            {
                // Skip the '
                Next();

                // Parse the character.
                char character = '\0';

                if(look == '\\')
                {
                    // Escape sequence.
                    Next();
                    switch(look)
                    {
                    case 'a':
                        character = '\a';
                        break;
                    case 'b':
                        character = '\b';
                        break;
                    case 'f':
                        character = '\f';
                        break;
                    case 't':
                        character = '\t';
                        break;
                    case 'r':
                        character = '\r';
                        break;
                    case 'n':
                        character = '\n';
                        break;
                    case 'v':
                        character = '\v';
                        break;
                    case '0':
                        character = '\0';
                        break;
                    case 'x':
                        break;
                    default:
                        character = (char)look;
                        break;
                    }

                    Next();
                    if(look != '\'')
                        Error("unclosed character literal.");
                }
                else if(look != '\'')
                {
                    character = (char)look;
                    Next();
                    if(look != '\'')
                        Error("unclosed character literal.");
                }
                else
                {
                    Error("empty character literal.");
                }

                // Skip the last \'
                Next();

                // Create the token.
                tokenValue = new IntegralToken(character, Token.CHARACTER, fileName, line, column);
                return true;
            }

            // Check for end-of-file.
            if(c1 == 0 && look == EOF)
            {
                tokenValue = new TokenValue(Token.EOF, fileName, line, column);
                return true;
            }

            // It must be an operator.
            int c2;
            if(c1 == 0)
            {
                c1 = look;
                Next();
                c2 = look;
            }
            else
            {
                c2 = look;
            }

            // Check for multi character operators.
            if(c2 == '=')
            {
                int tok = -1;
                switch(c1)
                {
                case '<':
                    tok = Token.LEQ;
                    break;
                case '>':
                    tok = Token.GEQ;
                    break;
                case '=':
                    tok = Token.EQ;
                    break;
                case '!':
                    tok = Token.NEQ;
                    break;
                case '+':
                    tok = Token.ADD_SET;
                    break;
                case '-':
                    tok = Token.SUB_SET;
                    break;
                case '*':
                    tok = Token.MUL_SET;
                    break;
                case '/':
                    tok = Token.DIV_SET;
                    break;
                case '%':
                    tok = Token.MOD_SET;
                    break;
                case '&':
                    tok = Token.AND_SET;
                    break;
                case '|':
                    tok = Token.OR_SET;
                    break;
                case '^':
                    tok = Token.XOR_SET;
                    break;
                default:
                    break;
                }

                if(tok > -1)
                {
                    // Advance.
                    Next();
                    tokenValue = new TokenValue(tok, fileName, line, column);
                    return true;
                }
            }
            else if(c1 == '<' && c2 == '<')
            {
                // Advance.
                Next();

                // The operator can be << and <<=;
                int tok = Token.LBITSHIFT;
                if(look == '=')
                {
                    tok = Token.LSHIFT_SET;
                    Next();
                }

                // Create the token.
                tokenValue = new TokenValue(tok, fileName, line, column);
                return true;
            }
            else if(c1 == '>' && c2 == '>')
            {
                // Advance.
                Next();

                // The operator can be >> or >>=;
                int tok = Token.RBITSHIFT;
                if(look == '=')
                {
                    tok = Token.RSHIFT_SET;
                    Next();
                }

                // Create the token.
                tokenValue = new TokenValue(tok, fileName, line, column);
                return true;
            }
            else if(c1 == '-' && c2 == '>')
            {
                // Advance.
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.ARROW, fileName, line, column);
                return true;
            }
            else if(c1 == '|' && c2 == '|')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.LOR, fileName, line, column);
                return true;
            }
            else if(c1 == '&' && c2 == '&')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.LAND, fileName, line, column);
                return true;
            }
            else if(c1 == c2 && c2 == '+')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.INCR, fileName, line, column);
                return true;
            }
            else if(c1 == c2 && c2 == '-')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.DECR, fileName, line, column);
                return true;
            }
            else if(c1 == '(' && c2 == '*')
            {
                // Check for function pointer start.
                long oldPosition = stream.Position;
                if(stream.ReadByte() == ')')
                {
                    // Consume the last ')'.
                    ++this.column;
                    Next();

                    tokenValue = new TokenValue(Token.FUNC_PTR, fileName, line, column);
                    return true;
                }
                else
                {
                    // Restore the position.
                    stream.Position = oldPosition;
                }
            }
            else if(c1 == '!')
            {
                // Create the token.
                tokenValue = new TokenValue(Token.LNOT, fileName, line, column);
                return true;
            }
            else if(c1 == '&')
            {
                tokenValue = new TokenValue(Token.BITAND, fileName, line, column);
                return true;
            }
            else if(c1 == '|')
            {
                tokenValue = new TokenValue(Token.BITOR, fileName, line, column);
                return true;
            }
            else if(c1 == '^')
            {
                tokenValue = new TokenValue(Token.BITXOR, fileName, line, column);
                return true;
            }
            else if(c1 == '~')
            {
                tokenValue = new TokenValue(Token.BITNOT, fileName, line, column);
                return true;
            }
            else if(c1 == '<')
            {
                return CheckGenericStart();
            }
            else if(c1 == '>' && genericStack.Count > 0)
            {
                int lastIndex = genericStack.Count - 1;
                int tokenId = genericStack[lastIndex];
                genericStack.RemoveAt(lastIndex);
                tokenValue = new TokenValue(tokenId, fileName, line, column);
                return true;
            }

            // Single character token. The token itself is his value.
            tokenValue = new TokenValue(c1, fileName, line, column);
            return true;
        }
コード例 #4
0
ファイル: Lexer.cs プロジェクト: MilkTool/chela
	    /** move on to next token.
	        @return false if positioned beyond tokens.
	        @throws IOException on input error.
	      */
	    public virtual bool advance ()
		{
            // Perform preprocessing.
            int c1 = Preprocess();

            // Store the line and column.
            int line = this.line;
            int column = this.column;

			// Read numbers.
			bool afterPeriod = false;
			if(c1 == 0 && look == '.')
			{
				afterPeriod = true;
				Next();
				if(!IsDigit(look))
				{
					// It was only a period.
					tokenValue = new TokenValue('.', fileName, line, column);
					return true;
				}		
			}
			
			// Check for numbers of the form ([0-9].[0-9]*(e[+-]?[0-9]+)?) | .[0-9]+(e[+-]?[0-9]+)?
            if(c1 == 0 && IsDigit(look))
			{
				ulong mantissa = 0;
				int exponent = 0;

                // Check for octal and hex.
                bool otherBase = false;
                if(look == '0' && !afterPeriod)
                {
                    Next();
                    if(look == 'x' || look == 'X')
                    {
                        // Set the other base flag.
                        otherBase = true;

                        // Hexadecimal number.
                        Next();
                        while(IsDigit(look) ||
                              (look >= 'a' && look <= 'f') ||
                              (look >= 'A' && look <= 'F'))
                        {
                            // Increase the mantissa.
                            mantissa *= 0x10;

                            // Append the digit.
                            if(look >= '0' && look <= '9')
                            {
                                mantissa += (ulong)(look - '0');
                            }
                            else if(look >= 'a' && look <= 'f')
                            {
                                mantissa += (ulong)(look - 'a' + 0xA);
                            }
                            else if(look >= 'A' && look <= 'F')
                            {
                                mantissa += (ulong)(look - 'A' + 0xA);
                            }

                            // Read the next digit.
                            Next();
                        }
                    }
                    else
                    {
                        // Set the other base flag.
                        otherBase = true;

                        // Octal number.
                        while(look >= '0' && look <= '7')
                        {
                            // Increase the mantissa.
                            mantissa *= 010u;

                            // Append the digit.
                            if(look >= '0' && look <= '7')
                                mantissa += (ulong)(look - '0');

                            // Read the next digit.
                            Next();
                        }

                        if(IsDigit(look) || look == '.' || look == 'e' || look == 'E')
                        {
                            // Unset the other base.
                            otherBase = false;

                            // Convert the octal into decimal.
                            mantissa = ulong.Parse(System.Convert.ToString((long)mantissa, 8));
                        }
                    }

                }

                if(!otherBase)
                {
    				while(IsDigit(look) || look == '.' || look == 'e' || look == 'E')
    				{
    					if(look == '.')
    					{
    						if(afterPeriod)
    							break;
    						afterPeriod = true;
    					}
    					else if(look == 'e' || look == 'E')
    					{
    						// Set the after period flag, to make sure we output a floating point number.
    						afterPeriod = true;
    						
    						// Read the exponent sign.
    						Next();
    						bool pos = true;
    						if(look == '+')
    						{
    							Next();
    						}
    						else if(look == '-')
    						{
    							pos = false;
    							Next();
    						}
    						
    						// Read the exponent.
    						int extraExponent = 0;
    						while(IsDigit(look))
    						{
    							extraExponent = extraExponent*10 + look - '0';
    							Next();
    						}

    						// Update the full exponent.
    						exponent += pos ? extraExponent : -extraExponent;

    						// Break of the loop.
    						break;
    					}
    					else
    					{
    						mantissa = mantissa*10 + (ulong)(look - '0');
    						if(afterPeriod)
    							exponent -= 1;
    					}
    					
    					// Read the next character.
    					Next();
    				}
                }
				
				// Read the suffix.
				int suffix = 0;
				if(IsAlpha(look))
				{
					suffix = look;
					Next();
				}
				
				// Create the token.
				if(afterPeriod)
				{
					// Check the suffix.
					int tok = Token.DOUBLE;
					if(suffix == 'f' || suffix == 'F')
						tok = Token.FLOAT;
					else if(suffix != 'd' && suffix != 'D' && suffix != 0)
					{
						// Invalid suffix.
						Error("Invalid floating point suffix '" + (char)suffix + "'");
					}
					
					// Make the value.
					double value = (double)mantissa;
					int exp = exponent;
					if(exp < 0) exp = -exp;
					for(int i = 0; i < exp; i++)
					{
						if(exponent > 0)
							value *= 10.0;
						else
							value *= 0.1;
					}

					// Create the token.
					tokenValue = new FloatingPointToken(value, tok, fileName, line, column);
				}
				else
				{
					if(suffix == 'u' || suffix == 'U')
					{
                        if(look == 'l' || look == 'L')
                        {
                            // ulong constant.
                            Next();

                            // Create the token.
                            tokenValue = new UIntegralToken(mantissa, Token.ULONG, fileName, line, column);
                        }
                        else
                        {
                            // Create the token.
    						tokenValue = new UIntegralToken(mantissa, Token.UINTEGER, fileName, line, column);
                        }
					}
					else
					{
                        if(suffix == 'l' || suffix == 'L')
                        {
                            // Create the token.
                            tokenValue = new IntegralToken((long)mantissa, Token.LONG, fileName, line, column);
                        }
                        else // Create the token, using the least capable type.
                        {
                            int tok = Token.INTEGER;
                            if(mantissa > 0x7FFFFFFF)
                                tok = Token.LONG;
                            tokenValue = new IntegralToken((long)mantissa, tok, fileName, line, column);
                        }
                    }
				}
				
				return true;
			}


            // Check for C-string.
            bool cstring = false;
            string identprefix = null;
            if(c1 == 0 && look == 'c')
            {
                Next();
                if(look == '"')
                    cstring = true;
                else
                    identprefix = "c";
            }

            // Read identifiers.
			if(c1 == 0 &&  (IsAlpha(look) || look == '_' || identprefix != null))
			{
				// Build the identifier.
				string identifier = ReadIdentifier(identprefix);

				// Boolean values has special treatment.
				if(identifier == "true")
				{
					tokenValue = new BoolToken(true, Token.BOOL, fileName, line, column);
					return true;
				}
				else if(identifier == "false")
				{
					tokenValue = new BoolToken(false, Token.BOOL, fileName, line, column);
					return true;
				}
				
				// Check for keywords.
				int key;
                if(!keywords.TryGetValue(identifier, out key))
                    key = Token.IDENTIFIER;
				
				// Create the token.
				tokenValue = new WordToken(identifier, key,
				                           fileName, line, column);				
				return true;
			}

            // Read raw identifiers.
            if(c1 == 0 && look == '@')
            {
                // Skip the @.
                Next();

                // Read the identifier.
                string identifier = ReadIdentifier();

                // Create the token.
                tokenValue = new WordToken(identifier, Token.IDENTIFIER,
                                           fileName, line, column);
                return true;
            }
			
			// Read literals.
			if(c1 == 0 && look == '"')
			{
                // Create the token.
				tokenValue = new WordToken(ReadLiteral(), cstring ? Token.CSTRING : Token.STRING, fileName, line, column);
				return true;
			}

            // Read characters.
            if(c1 == 0 && look == '\'')
            {
                // Skip the '
                Next();

                // Parse the character.
                char character = '\0';

                if(look == '\\')
                {
                    // Escape sequence.
                    Next();
                    switch(look)
                    {
                    case 'a':
                        character = '\a';
                        break;
                    case 'b':
                        character = '\b';
                        break;
                    case 'f':
                        character = '\f';
                        break;
                    case 't':
                        character = '\t';
                        break;
                    case 'r':
                        character = '\r';
                        break;
                    case 'n':
                        character = '\n';
                        break;
                    case 'v':
                        character = '\v';
                        break;
                    case '0':
                        character = '\0';
                        break;
                    case 'x':
                        break;
                    default:
                        character = (char)look;
                        break;
                    }

                    Next();
                    if(look != '\'')
                        Error("unclosed character literal.");
                }
                else if(look != '\'')
                {
                    character = (char)look;
                    Next();
                    if(look != '\'')
                        Error("unclosed character literal.");
                }
                else
                {
                    Error("empty character literal.");
                }

                // Skip the last \'
                Next();

                // Create the token.
                tokenValue = new IntegralToken(character, Token.CHARACTER, fileName, line, column);
                return true;
            }

			// Check for end-of-file.
			if(c1 == 0 && look == EOF)
			{
				tokenValue = new TokenValue(Token.EOF, fileName, line, column);
				return true;
			}
			
			// It must be an operator.
            int c2;
            if(c1 == 0)
            {
			    c1 = look;
			    Next();
			    c2 = look;
            }
            else
            {
                c2 = look;
            }
			
			// Check for multi character operators.
			if(c2 == '=')
			{
				int tok = -1;
				switch(c1)
				{
				case '<':
					tok = Token.LEQ;
					break;
				case '>':
					tok = Token.GEQ;
					break;
				case '=':
					tok = Token.EQ;
					break;
				case '!':
					tok = Token.NEQ;
					break;
				case '+':
					tok = Token.ADD_SET;
					break;
				case '-':
					tok = Token.SUB_SET;
					break;
				case '*':
					tok = Token.MUL_SET;
					break;
				case '/':
					tok = Token.DIV_SET;
					break;
				case '%':
					tok = Token.MOD_SET;
					break;
				case '&':
					tok = Token.AND_SET;
					break;
				case '|':
					tok = Token.OR_SET;
					break;
				case '^':
					tok = Token.XOR_SET;
					break;
				default:
					break;
				}
				
				if(tok > -1)
				{
					// Advance.
					Next();
					tokenValue = new TokenValue(tok, fileName, line, column);
					return true;
				}
			}
			else if(c1 == '<' && c2 == '<')
			{
				// Advance.
				Next();
				
				// The operator can be << and <<=;
				int tok = Token.LBITSHIFT;
				if(look == '=')
				{
					tok = Token.LSHIFT_SET;
					Next();
				}
				
				// Create the token.
				tokenValue = new TokenValue(tok, fileName, line, column);
				return true;
			}
			else if(c1 == '>' && c2 == '>')
			{
				// Advance.
				Next();
				
				// The operator can be >> or >>=;
				int tok = Token.RBITSHIFT;
				if(look == '=')
				{
					tok = Token.RSHIFT_SET;
					Next();
				}
				
				// Create the token.
				tokenValue = new TokenValue(tok, fileName, line, column);
				return true;
			}
			else if(c1 == '-' && c2 == '>')
			{
				// Advance.
				Next();
				
				// Create the token.
				tokenValue = new TokenValue(Token.ARROW, fileName, line, column);
				return true;
			}
			else if(c1 == '|' && c2 == '|')
			{
				// Advance
				Next();
				
				// Create the token.
				tokenValue = new TokenValue(Token.LOR, fileName, line, column);
				return true;
			}
			else if(c1 == '&' && c2 == '&')
			{
				// Advance
				Next();
				
				// Create the token.
				tokenValue = new TokenValue(Token.LAND, fileName, line, column);
				return true;
			}
            else if(c1 == c2 && c2 == '+')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.INCR, fileName, line, column);
                return true;
            }
            else if(c1 == c2 && c2 == '-')
            {
                // Advance
                Next();

                // Create the token.
                tokenValue = new TokenValue(Token.DECR, fileName, line, column);
                return true;
            }
            else if(c1 == '(' && c2 == '*')
            {
                // Check for function pointer start.
                long oldPosition = stream.Position;
                if(stream.ReadByte() == ')')
                {
                    // Consume the last ')'.
                    ++this.column;
                    Next();

                    tokenValue = new TokenValue(Token.FUNC_PTR, fileName, line, column);
                    return true;
                }
                else
                {
                    // Restore the position.
                    stream.Position = oldPosition;
                }
            }
            else if(c1 == '!')
            {
                // Create the token.
                tokenValue = new TokenValue(Token.LNOT, fileName, line, column);
                return true;
            }
            else if(c1 == '&')
            {
                tokenValue = new TokenValue(Token.BITAND, fileName, line, column);
                return true;
            }
            else if(c1 == '|')
            {
                tokenValue = new TokenValue(Token.BITOR, fileName, line, column);
                return true;
            }
            else if(c1 == '^')
            {
                tokenValue = new TokenValue(Token.BITXOR, fileName, line, column);
                return true;
            }
            else if(c1 == '~')
            {
                tokenValue = new TokenValue(Token.BITNOT, fileName, line, column);
                return true;
            }
            else if(c1 == '<')
            {
                return CheckGenericStart();
            }
            else if(c1 == '>' && genericStack.Count > 0)
            {
                int lastIndex = genericStack.Count - 1;
                int tokenId = genericStack[lastIndex];
                genericStack.RemoveAt(lastIndex);
                tokenValue = new TokenValue(tokenId, fileName, line, column);
                return true;
            }


			// Single character token. The token itself is his value.
			tokenValue = new TokenValue(c1, fileName, line, column);
			return true;
		}
コード例 #5
0
ファイル: Lexer.cs プロジェクト: MilkTool/chela
		public Lexer (string fileName, Stream stream)
		{
			this.fileName = fileName;
			this.stream = stream;
			this.line = 1;
			this.column = 0;
			this.tokenValue = null;
			this.keywords = new Dictionary<string, int> ();
            this.definitions = new Dictionary<string, bool> ();
            this.preprocessorStates = new List<bool> ();
            this.ignoring = false;
            this.ignoreCount = 0;
            this.hasBeenTruePrep = false;
            this.start = true;
            this.genericStack = new List<int> ();

			// Look the first character.
			Next();
			
			// Set the keywords.
			this.keywords.Add("bool", Token.KBOOL);
			this.keywords.Add("byte", Token.KBYTE);
			this.keywords.Add("sbyte", Token.KSBYTE);
			this.keywords.Add("char", Token.KCHAR);
			this.keywords.Add("uchar", Token.KUCHAR);
			this.keywords.Add("double", Token.KDOUBLE);
			this.keywords.Add("float", Token.KFLOAT);
			this.keywords.Add("int", Token.KINT);
			this.keywords.Add("uint", Token.KUINT);
			this.keywords.Add("long", Token.KLONG);
			this.keywords.Add("ulong", Token.KULONG);
			this.keywords.Add("short", Token.KSHORT);
			this.keywords.Add("ushort", Token.KUSHORT);
			this.keywords.Add("object", Token.KOBJECT);
			this.keywords.Add("string", Token.KSTRING);
			this.keywords.Add("size_t", Token.KSIZE_T);
            this.keywords.Add("ptrdiff_t", Token.KPTRDIFF_T);
			this.keywords.Add("void", Token.KVOID);
            this.keywords.Add("vec2", Token.KVEC2);
            this.keywords.Add("vec3", Token.KVEC3);
            this.keywords.Add("vec4", Token.KVEC4);
            this.keywords.Add("dvec2", Token.KDVEC2);
            this.keywords.Add("dvec3", Token.KDVEC3);
            this.keywords.Add("dvec4", Token.KDVEC4);
            this.keywords.Add("ivec2", Token.KIVEC2);
            this.keywords.Add("ivec3", Token.KIVEC3);
            this.keywords.Add("ivec4", Token.KIVEC4);
            this.keywords.Add("bvec2", Token.KBVEC2);
            this.keywords.Add("bvec3", Token.KBVEC3);
            this.keywords.Add("bvec4", Token.KBVEC4);

            this.keywords.Add("mat2"  , Token.KMAT2);
            this.keywords.Add("mat2x3", Token.KMAT2x3);
            this.keywords.Add("mat2x4", Token.KMAT2x4);
            this.keywords.Add("mat3x2", Token.KMAT3x2);
            this.keywords.Add("mat3"  , Token.KMAT3);
            this.keywords.Add("mat3x4", Token.KMAT3x4);
            this.keywords.Add("mat4x2", Token.KMAT4x2);
            this.keywords.Add("mat4x3", Token.KMAT4x3);
            this.keywords.Add("mat4"  , Token.KMAT4);

            this.keywords.Add("dmat2"  , Token.KDMAT2);
            this.keywords.Add("dmat2x3", Token.KDMAT2x3);
            this.keywords.Add("dmat2x4", Token.KDMAT2x4);
            this.keywords.Add("dmat3x2", Token.KDMAT3x2);
            this.keywords.Add("dmat3"  , Token.KDMAT3);
            this.keywords.Add("dmat3x4", Token.KDMAT3x4);
            this.keywords.Add("dmat4x2", Token.KDMAT4x2);
            this.keywords.Add("dmat4x3", Token.KDMAT4x3);
            this.keywords.Add("dmat4"  , Token.KDMAT4);

            this.keywords.Add("imat2"  , Token.KIMAT2);
            this.keywords.Add("imat2x3", Token.KIMAT2x3);
            this.keywords.Add("imat2x4", Token.KIMAT2x4);
            this.keywords.Add("imat3x2", Token.KIMAT3x2);
            this.keywords.Add("imat3"  , Token.KIMAT3);
            this.keywords.Add("imat3x4", Token.KIMAT3x4);
            this.keywords.Add("imat4x2", Token.KIMAT4x2);
            this.keywords.Add("imat4x3", Token.KIMAT4x3);
            this.keywords.Add("imat4"  , Token.KIMAT4);

            this.keywords.Add("typedef", Token.KTYPEDEF);
			this.keywords.Add("class", Token.KCLASS);
            this.keywords.Add("delegate", Token.KDELEGATE);
            this.keywords.Add("event", Token.KEVENT);
            this.keywords.Add("interface", Token.KINTERFACE);
            this.keywords.Add("enum", Token.KENUM);
			this.keywords.Add("namespace", Token.KNAMESPACE);
			this.keywords.Add("struct", Token.KSTRUCT);
            this.keywords.Add("operator", Token.KOPERATOR);
			
			this.keywords.Add("public", Token.KPUBLIC);
			this.keywords.Add("internal", Token.KINTERNAL);
			this.keywords.Add("protected", Token.KPROTECTED);
			this.keywords.Add("private", Token.KPRIVATE);
			this.keywords.Add("extern", Token.KEXTERN);
			this.keywords.Add("kernel", Token.KKERNEL);
			this.keywords.Add("virtual", Token.KVIRTUAL);
			this.keywords.Add("static", Token.KSTATIC);
			this.keywords.Add("override", Token.KOVERRIDE);
            this.keywords.Add("abstract", Token.KABSTRACT);
            this.keywords.Add("sealed", Token.KSEALED);
            this.keywords.Add("const", Token.KCONST);
            this.keywords.Add("readonly", Token.KREADONLY);
            this.keywords.Add("unsafe", Token.KUNSAFE);
            this.keywords.Add("partial", Token.KPARTIAL);

            this.keywords.Add("__cdecl", Token.KCDECL);
            this.keywords.Add("__stdcall", Token.KSTDCALL);
            this.keywords.Add("__apicall", Token.KAPICALL);

			this.keywords.Add("break", Token.KBREAK);
			this.keywords.Add("case", Token.KCASE);
            this.keywords.Add("catch", Token.KCATCH);
			this.keywords.Add("continue", Token.KCONTINUE);
            this.keywords.Add("default", Token.KDEFAULT);
			this.keywords.Add("do", Token.KDO);
			this.keywords.Add("else", Token.KELSE);
            this.keywords.Add("finally", Token.KFINALLY);
            this.keywords.Add("fixed", Token.KFIXED);
			this.keywords.Add("for", Token.KFOR);
			this.keywords.Add("foreach", Token.KFOREACH);
            this.keywords.Add("goto", Token.KGOTO);
			this.keywords.Add("if", Token.KIF);
            this.keywords.Add("in", Token.KIN);
            this.keywords.Add("lock", Token.KLOCK);
			this.keywords.Add("return", Token.KRETURN);
			this.keywords.Add("while", Token.KWHILE);
            this.keywords.Add("using", Token.KUSING);
            this.keywords.Add("switch", Token.KSWITCH);
            this.keywords.Add("throw", Token.KTHROW);
            this.keywords.Add("try", Token.KTRY);
			
			this.keywords.Add("null", Token.KNULL);
            this.keywords.Add("this", Token.KTHIS);
            this.keywords.Add("base", Token.KBASE);

			this.keywords.Add("bitand", Token.BITAND);
			this.keywords.Add("bitor", Token.BITOR);
			this.keywords.Add("compl", Token.BITNOT);
			this.keywords.Add("xor", Token.BITXOR);
			this.keywords.Add("and", Token.LAND);
			this.keywords.Add("or", Token.LOR);
			this.keywords.Add("not", Token.LNOT);
            this.keywords.Add("sizeof", Token.SIZEOF);
            this.keywords.Add("typeof", Token.TYPEOF);
            this.keywords.Add("is", Token.IS);
            this.keywords.Add("as", Token.AS);
			this.keywords.Add("new", Token.NEW);
            this.keywords.Add("heapalloc", Token.HEAPALLOC);
            this.keywords.Add("stackalloc", Token.STACKALLOC);
			this.keywords.Add("delete", Token.KDELETE);
            this.keywords.Add("reinterpret_cast", Token.REINTERPRET_CAST);
            this.keywords.Add("checked", Token.CHECKED);
            this.keywords.Add("unchecked", Token.UNCHECKED);

            this.keywords.Add("ref", Token.KREF);
            this.keywords.Add("out", Token.KOUT);
            this.keywords.Add("params", Token.KPARAMS);
		}
コード例 #6
0
ファイル: Lexer.cs プロジェクト: MilkTool/chela
        private bool CheckGenericStart()
        {
            // Store the old state.
            int oldLine = line;
            int oldColumn = column;
            int oldLook = look;
            long oldPosition = stream.Position;

            // Start with GENERIC_START.
            int token = Token.GENERIC_START;

            // Make sure there are only comments, ',', '.', '*', '<' and identifiers
            // before hitting '>'.
            int genericCheckCount = 1;
            bool foundDot = false;
            for(;;)
            {
                // Preprocess
                int firstLook = Preprocess();
                if(firstLook == 0)
                    firstLook = look;

                // If hit EOF, end loop.
                if(firstLook == EOF)
                {
                    token = '<';
                    break;
                }

                if(genericCheckCount == 0)
                {
                    foundDot = firstLook == '.';
                    break;
                }

                // Check for identifiers.
                if(IsAlpha(firstLook) || firstLook == '_')
                {
                    // Identifier.
                    Next();
                    while(IsAlpha(look) || IsDigit(look) || look == '_')
                        Next();

                    // Consume the next token.
                    continue;
                }

                // Check for end single character tokens.
                if(firstLook == ',' || firstLook == '.' || firstLook == '*')
                {
                    // Consume the token.
                    Next();
                }
                else if(firstLook == '<')
                {
                    // Consume the token.
                    Next();
                    ++genericCheckCount;

                }
                else if(firstLook == '>')
                {
                    // Consume the token.
                    Next();
                    --genericCheckCount;
                    if(genericCheckCount == 0)
                        continue;
                }
                else
                {
                    // This is not a generic.
                    token = '<';
                    break;
                }

            }

            // Restore the old state.
            line = oldLine;
            column = oldColumn;
            look = oldLook;
            stream.Seek(oldPosition, SeekOrigin.Begin);

            // Increase the generic count.
            if(token == Token.GENERIC_START)
            {
                if(foundDot)
                {
                    token = Token.GENERIC_START_DOT;
                    genericStack.Add(Token.GENERIC_END_DOT);
                }
                else
                {
                    genericStack.Add(Token.GENERIC_END);
                }
            }

            // Return the token.
            tokenValue = new TokenValue(token, fileName, line, column);
            return true;
        }