예제 #1
0
파일: Token.cs 프로젝트: D-Eijkelenboom/DP2
 //public Token() { }
 public Token(int linenr, int posnr, TokenType token, string value, int level, Token partner)
 {
     LineNr = linenr;
     PositionNr = posnr;
     Type = token;
     Value = value;
     Level = level;
     Partner = partner;
 }
예제 #2
0
        private List<Token> tokenize(string[] lines)
        {
            int posNr = 0;
            int lineNr = 0;
            int level = 0;

            foreach(string line in lines)
            {
                string cleanLine = Regex.Replace(line, @"\t|\n|\r", "");
                lineNr++;

                // escape new line
                if (cleanLine == "")
                    continue;

                string[] words = cleanLine.Split(' ');

                for (int i = 0; i < words.Length; i++)
                {
                    switch (words[i])
                    {
                        case "print":
                            tokens.Add(new Token(lineNr, posNr, TokenType.PRINT, "print", level, null));
                            break;
                        case "for":
                            tokens.Add(new Token(lineNr, posNr, TokenType.FOR, "for", level, null));
                            break;
                        case "while":
                            tokens.Add(new Token(lineNr, posNr, TokenType.WHILE, "while", level, null));
                            break;
                        case "if":
                            tokens.Add(new Token(lineNr, posNr, TokenType.IF, "if", level, null));
                            ifTokens.Push(tokens.LastOrDefault());
                            break;
                        case "else":
                            if (ifTokens.Count == 0)
                            {
                                Console.WriteLine("Error: IF statement not found");
                            }
                            else
                            {
                                Token partner = ifTokens.Pop();
                                tokens.Add(new Token(lineNr, posNr, TokenType.ELSE, "else", level, partner));
                                partner.Partner = tokens.LastOrDefault();
                            }
                            break;
                        case "{":
                            level++;
                            Token token = new Token(lineNr, posNr, TokenType.BRACKETOPEN, "{", level, null);
                            needsClosure.Push(token);
                            tokens.Add(token);
                            break;
                        case "}":
                            if (needsClosure.Peek().Type != TokenType.BRACKETOPEN)
                            {
                                Console.WriteLine("Error: } not found");
                            }
                            else
                            {
                                level--;
                                Token partner = needsClosure.Pop();
                                tokens.Add(new Token(lineNr, posNr, TokenType.BRACKETCLOSE, "}", level, partner));
                                partner.Partner = tokens.LastOrDefault();

                            }
                            break;
                        case "(":
                            tokens.Add(new Token(lineNr, posNr, TokenType.ELIPSISOPEN, "(", level, null));
                            needsClosure.Push(tokens.LastOrDefault());

                            break;
                        case ")":
                            if (needsClosure.Peek().Type != TokenType.ELIPSISOPEN)
                            {
                                Console.WriteLine("Error: ( not found");
                            }
                            else
                            {
                                Token partner = needsClosure.Pop();
                                tokens.Add(new Token(lineNr, posNr, TokenType.ELIPSISCLOSE, ")", level, partner));
                                partner.Partner = tokens.LastOrDefault();
                            }
                            break;
                        case ";":
                            tokens.Add(new Token(lineNr, posNr, TokenType.SEMICOLON, ";", level, null));
                            break;
                        case "=":
                            tokens.Add(new Token(lineNr, posNr, TokenType.EQUALS, "=", level, null));
                            break;
                        case "==":
                            tokens.Add(new Token(lineNr, posNr, TokenType.COMPARE, "==", level, null));
                            break;
                        case "<=":
                            tokens.Add(new Token(lineNr, posNr, TokenType.LESSEREQUALS, "<=", level, null));
                            break;
                        case ">=":
                            tokens.Add(new Token(lineNr, posNr, TokenType.GREATEREQUALS, ">=", level, null));
                            break;
                        case "!=":
                            tokens.Add(new Token(lineNr, posNr, TokenType.NOTEQUALS, "!=", level, null));
                            break;
                        case "+":
                            tokens.Add(new Token(lineNr, posNr, TokenType.PLUS, "+", level, null));
                            break;
                        case "-":
                            tokens.Add(new Token(lineNr, posNr, TokenType.MINUS, "-", level, null));
                            break;
                        case "*":
                            tokens.Add(new Token(lineNr, posNr, TokenType.MULTIPLY, "*", level, null));
                            break;
                        case "/":
                            tokens.Add(new Token(lineNr, posNr, TokenType.DIVIDE, "/", level, null));
                            break;
                        case "++":
                            tokens.Add(new Token(lineNr, posNr, TokenType.INCREMENT, "++", level, null));
                            break;
                        case "--":
                            tokens.Add(new Token(lineNr, posNr, TokenType.DECREMENT, "--", level, null));
                            break;
                        default:
                            string temp = Regex.Replace(words[i], @";", "");
                            int value;
                            if (int.TryParse(temp, out value))
                            {
                                tokens.Add(new Token(lineNr, posNr, TokenType.NUMBER, value.ToString(), level, null));
                            }
                            else
                            {
                                tokens.Add(new Token(lineNr, posNr, TokenType.IDENTIFIER, temp, level, null));
                            }
                            break;
                    }
                    posNr = posNr + words[i].Length + 1;
                }
            }
            if(needsClosure.Count != 0)
            {
                Console.WriteLine("Compile error: \"} or ) \" missing!");
            }

            connectTokens();

            return tokens;
        }
예제 #3
0
        public static Color GetColor(Token token)
        {
            if (!s_colors.ContainsKey(token.Type)) return Color.Black;

            return s_colors[token.Type];
        }
예제 #4
0
 public void AddAstStringLiteralValueNode(Token t)
 {
     var node = new AstStringLiteralExpression(t.Attribute);
     PushNode(node);
 }
예제 #5
0
 public void AddAstIntegerValueNode(Token t)
 {
     var node = new AstIntegerValueExpression(t.Attribute);
     PushNode(node);
 }
예제 #6
0
 public void AddAstIdNode(Token t)
 {
     var node = new AstIdExpression(t.Attribute);
     PushNode(node);
 }
예제 #7
0
        private Token SendBlockClosingTokens(Token currToken)
        {
            if (currIndentationLevel > 0)
            {
                var result = new Token(TokenType.BLOCK_END, currIndentationLevel.ToString());
                LeaveBlock();
                return result;
            }

            return currToken;
        }
예제 #8
0
        private Token ReadBlockStart(Token currToken)
        {
            currToken = GetNextNotSpace();
            if (currToken.Type != TokenType.LINE_END)
            {
                return GetErrorToken("Expected line ending");
            }

            EnterBlock();
            try
            {
                ReadIndents();
            }
            catch (IndentationException e)
            {
                return GetErrorToken(e.Message);
            }

            return new Token(TokenType.BLOCK_START, currIndentationLevel.ToString());
        }
예제 #9
0
        private Token CheckIndentation(Token currToken)
        {
            ushort counter = 0;
            Token t;
            do
            {
                t = baseScanner.GetForwardToken();
                if (t.Type == TokenType.SPACE)
                {
                    baseScanner.GetNextToken();
                    ++counter;
                }
                else if (t.Type == TokenType.LINE_END)
                {
                    baseScanner.GetNextToken();
                    counter = 0;
                }
            } while (t.Type == TokenType.SPACE || t.Type == TokenType.LINE_END); // empty line - skip

            if (counter == GetSpacesCount())
            {
                nextTokens.Enqueue(baseScanner.GetNextToken()); // right level - save next token
                return currToken;
            }

            if ((counter < GetSpacesCount()) && ((counter % indentSize) == 0)) // level decreased - if more than 1 level - store in stack and return blockend
            {
                var diff = Convert.ToInt16((GetSpacesCount() - counter) / indentSize);
                for (var i = 0; i < diff; ++i)
                {
                    counter -= indentSize;

                    var result = new Token(TokenType.BLOCK_END, currIndentationLevel.ToString());
                    LeaveBlock();
                    nextTokens.Enqueue(result);
                }
                return currToken;
            }

            return GetErrorToken("Wrong indentation level."); // wrong level - return error
        }
예제 #10
0
        public Token scan()
        {
            for (; ; readch())
            {
                if (peek == ' ' || peek == '\t')
                    continue;
                else if (peek == '\n')
                    line++;
                else
                    break;
            }

            if (peek == '/')
            {
                readch();
                if (peek == '/')
                {
                    while (peek != '\n')
                        readch();
                    line++;
                }
                else if (peek == '*')
                {
                    readch();
                    while (true)
                    {
                        if (peek == '*')
                        {
                            readch();
                            if (peek == '/')
                                break;
                        }
                        else if (peek == 65535)
                            return new Word("_THE_END_", Tag._THE_END_);
                        else
                            readch();
                    }
                }
                else
                    return new Token('/');
            }

            /*            switch (peek)
                        {
                            case '&':
                                if (readch('&')) return Word.and; else return new Token('&');
                            case '|':
                                if (readch('|')) return Word.or; else return new Token('|');
                            case '=':
                                if (readch('=')) return Word.eq; else return new Token('=');
                            case '!':
                                if (readch('=')) return Word.ne; else return new Token('!');
                            case '<':
                                if (readch('=')) return Word.le; else return new Token('<');
                            case '>':
                                if (readch('=')) return Word.ge; else return new Token('>');
                        }
            */
            if (Char.IsDigit(peek))
            {
                int v = 0;
                do
                {
                    v = 10 * v + peek;
                    readch();
                } while (Char.IsDigit(peek));

                if (peek != '.') return new Num(v);
                double x = v;
                double d = 10.0;
                for (; ; )
                {
                    readch();
                    if (!Char.IsDigit(peek))
                        break;
                    x = x + peek / d;
                    d *= 10;
                }

                return new Real(x);
            }

            if (Char.IsLetter(peek) || peek == '.')
            {
                StringBuilder b = new StringBuilder();
                do
                {
                    if (peek == '.')
                        b.Append('_');
                    else
                        b.Append(peek);
                    readch();
                } while (Char.IsLetter(peek) || peek == '_');

                string s = b.ToString();
                Word w = (Word)words[s];
                if (w != null) return w;
                w = new Word(s, Tag.ID);
                words.Add(s, w);

                return w;
            }

            Token token = new Token(peek);
            peek = ' ';
            return token;
        }
예제 #11
0
 public static bool IsCorrectToken(Token t)
 {
     return (t.Type != TokenType.EOF) && (t.Type != TokenType.ERROR);
 }
예제 #12
0
 public virtual bool IsMatch(Token currentToken)
 {
     return false;
 }
예제 #13
0
 public virtual void Compile(Token currentToken)
 {
 }