public override Token Match(Scanner scanner)
        {
            if (!char.IsLetterOrDigit(scanner.Current) && scanner.Current != '_')
            {
                return(Token.Invalid);
            }

            TokenPos pos        = scanner.Pos;
            int      beginIndex = scanner.ReadedCount;
            int      identCount = 0;

            while (true)
            {
                char c = scanner.Current;
                if (char.IsLetterOrDigit(c) || c == '_')
                {
                    scanner.Consume(1);
                    identCount++;
                    continue;
                }

                break;
            }

            return(new Token(ID, scanner.Source.Substring(beginIndex, identCount), pos));
        }
Пример #2
0
        public override Token Match(Scanner scanner)
        {
            if (scanner.Peek(0) != '/' || scanner.Peek(1) != '/')
            {
                return(Token.Invalid);
            }

            TokenPos pos = scanner.Pos;

            scanner.Consume(2);

            StringBuilder comment = new StringBuilder();

            while (true)
            {
                char c = scanner.Peek(0);
                scanner.Consume(1);

                if (c == '\0')
                {
                    break;
                }

                if (c == '\n')
                {
                    scanner.ChangeLine();
                    break;
                }
                comment.Append(c);
            }

            return(new Token(ID, comment.ToString(), pos));
        }
        public override Token Match(Scanner scanner)
        {
            if (scanner.Peek(0) != '/' || scanner.Peek(1) != '1')
            {
                return(Token.Invalid);
            }

            TokenPos pos = scanner.Pos;

            scanner.Consume(2);
            StringBuilder comment = new StringBuilder();

            while (scanner.Peek(0) != '*' || scanner.Peek(2) != '/')
            {
                char c = scanner.Peek(0);

                comment.Append(c);
                scanner.Consume(1);
            }

            scanner.Consume(2);


            return(new Token(ID, comment.ToString(), pos));
        }
Пример #4
0
        public override Token Match(Scanner scanner)
        {
            if (scanner.Current != '"' || scanner.Current != '\'')
            {
                return(Token.Invalid);
            }

            TokenPos pos = scanner.Pos;

            scanner.Consume(1);
            _Content.Clear();

            bool encounterEscapeCode = false;

            while (true)
            {
                if (encounterEscapeCode)
                {
                    switch (scanner.Current)
                    {
                    case 'n':
                        _Content.Append('\n');
                        break;

                    case 'r':
                        _Content.Append('\r');
                        break;

                    case 't':
                        _Content.Append('\t');
                        break;

                    case '\'':
                        _Content.Append(scanner.Current);
                        break;

                    default:
                        _Content.Append('\\');
                        break;
                    }
                    encounterEscapeCode = false;
                }
                else if (scanner.Current == '\\')
                {
                    encounterEscapeCode = true;
                }


                if (scanner.Current == '"' || scanner.Current == '\'')
                {
                    break;
                }

                scanner.Consume(1);
            }


            return(new Token(ID, _Content.ToString(), pos));
        }
Пример #5
0
        public override Token Match(Scanner scanner)
        {
            if (!(char.IsDigit(scanner.Current) || scanner.Current == '-' && char.IsDigit(scanner.Peek(1))))
            {
                return(Token.Invalid);
            }

            TokenPos pos = scanner.Pos;

            _Content.Clear();
            bool encounterDot = false;

            while (true)
            {
                _Content.Append(scanner.Current);
                scanner.Consume(1);
                if (!char.IsDigit(scanner.Current))
                {
                    if (scanner.Current == '.')
                    {
                        encounterDot = true;
                    }
                    break;
                }
            }

            // if encounter Dot is true that means it's a float number
            if (encounterDot)
            {
                int index = 0;
                while (true)
                {
                    _Content.Append(scanner.Current);
                    scanner.Consume(1);
                    if (!char.IsDigit(scanner.Current))
                    {
                        if (index == 0)
                        {
                            throw new Exception("illegal float number: " + pos.ToString());
                        }

                        break;
                    }

                    index++;
                }
            }

            return(new Token(ID, _Content.ToString(), pos));
        }
Пример #6
0
        public override Token Match(Scanner scanner)
        {
            var signLentgh = Sign.Length;

            for (int i = 0; i < signLentgh; i++)
            {
                if (Sign[i] != scanner.Peek(i))
                {
                    return(Token.Invalid);
                }
            }

            TokenPos pos = scanner.Pos;

            scanner.Consume(signLentgh);

            return(new Token(ID, Sign, pos));
        }
Пример #7
0
        public override Token Match(Scanner scanner)
        {
            // 在windows系统里面, \r\n为换行
            // 在Unix/Mac下,  \n为换行

            TokenPos pos = TokenPos.InitPos;

            var content = new StringBuilder();

            while (true)
            {
                var c = scanner.Peek(0);
                if (c == '\n')
                {
                    if (pos.Equals(TokenPos.InitPos))
                    {
                        pos = scanner.Pos;
                    }
                    content.Append(c);
                    scanner.ChangeLine();
                    //scanner.Consume(1);
                    continue;
                }
                else if (c == '\r')
                {
                    content.Append(c);
                    scanner.Consume(1);
                    continue;
                }

                break;
            }

            if (pos.Equals(TokenPos.InitPos))
            {
                return(Token.Invalid);
            }

            return(new Token(ID, content.ToString(), pos));
        }
Пример #8
0
 public Token(int id, string value, TokenPos pos)
 {
     ID    = id;
     Value = value;
     Pos   = pos;
 }
Пример #9
0
 public bool Equals(TokenPos pos)
 {
     return(Col == pos.Col && Line == pos.Line && FileName == pos.FileName);
 }