예제 #1
0
        /// <summary>
        /// Scans the input stream and returns scanned <see cref="Token"/> objects.
        /// </summary>
        /// <returns></returns>
        public TokenSequence Scan()
        {
            TokenSequence tokens = new TokenSequence();

            //
            // Scan indents in the beginning
            if(!sourceReader.Empty)
                ScanIndent(tokens);

            //
            // Main loop
            while(!sourceReader.Empty)
            {
                ScanInternal(tokens);
            } // while

            //
            // Terminate statements, if any
            if(tokens.Count > 0 && tokens.Last.Type != TokenType.EndStatement)
                tokens.Add(new Token(TokenType.EndStatement, null));

            //
            // Now postprocess what we have scanned so far and
            // remove extraneous Indents & stuff
            tokens = PostProcessTokens(new TokenSequence(tokens));

            return tokens;
        }
        public void Last()
        {
            TokenSequence sequence = new TokenSequence();
            sequence.Add(new Token(TokenType.BeginBlock, null));
            sequence.Add(new Token(TokenType.Colon, null));
            sequence.Add(new Token(TokenType.Comma, null));

            Assert.AreEqual(new Token(TokenType.Comma, null), sequence.Last);
        }
        public void AddAndCount()
        {
            TokenSequence sequence = new TokenSequence();
            sequence.Add(new Token(TokenType.BeginBlock, null));
            sequence.Add(new Token(TokenType.Colon, null));
            sequence.Add(new Token(TokenType.Comma, null));

            Assert.AreEqual(3, sequence.Count);
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="TokenSequence"/> class.
 /// </summary>
 /// <param name="tokens"></param>
 public TokenSequence(TokenSequence tokens)
 {
     foreach(Token token in tokens.tokens)
         Add(token);
 }
예제 #5
0
        private void ScanInternal(TokenSequence tokens)
        {
            char lookahead = sourceReader.Next;

            if(char.IsDigit(lookahead) || lookahead == '-')
                tokens.Add(ScanIntegerConstant());
            else if(lookahead == '"')
                tokens.Add(ScanStringConstant());
            else if(char.IsLetter(lookahead))
                tokens.Add(ScanSymbolOrKeyword());
            else if(lookahead == ':')
                tokens.Add(ScanColon());
            else if(lookahead == '=')
                tokens.Add(ScanPropertyAssignment());
            else if(lookahead == '[')
                tokens.Add(ScanLeftSquareBracket());
            else if(lookahead == ']')
                tokens.Add(ScanRightSquareBracket());
            else if(lookahead == '{')
                tokens.Add(ScanLeftBrace());
            else if(lookahead == '}')
                tokens.Add(ScanRightBrace());
            else if(lookahead == '(')
                tokens.Add(ScanLeftBracket());
            else if(lookahead == ')')
                tokens.Add(ScanRightBracket());
            else if(lookahead == ',')
                tokens.Add(ScanComma());
            else if(lookahead == '/')
                ScanComment();
            else if(lookahead == '\r')
            {
                tokens.Add(new Token(TokenType.EndStatement, new Location(sourceReader.Line, sourceReader.Column + 1)));

                sourceReader.ReadNext();
                sourceReader.ReadNext();

                ScanIndent(tokens);
            } // else if
            else if(!sourceReader.Empty)
                sourceReader.ReadNext();
        }
예제 #6
0
        private void ScanIndent(TokenSequence tokens)
        {
            int indent = 0;

            while(!sourceReader.Empty && (sourceReader.Next == ' ' || sourceReader.Next == '\t' ))
            {
                if(sourceReader.Next == ' ')
                    indent += 1;
                else if(sourceReader.Next == '\t')
                    indent += 4;

                sourceReader.ReadNext();
            } // while

            if(indent == 0)
                return;

            tokens.Add(new Token(TokenType.Indent, new StringBuilder().Insert(0, " ", indent).ToString(), 
                new Location(sourceReader.Line, sourceReader.Column + 1)));

            //
            // Return if we reached the end of the line - nothing to do here
            /*if(!sourceReader.Empty && sourceReader.Next == '\r')
                return;

            if(indents.Count == 0 || indents.Peek() < indent)
            {
                if(indents.Count == 0 || tokens.Count == 0)
                {
                    indents.Push(indent);
                    tokens.Add(new Token(TokenType.BeginBlock, null));
                } // if
                else
                {
                    if(tokens.Last.Type == TokenType.BeginBlock)
                    {
                        if(indents.Peek() != indent)
                        {
                            indents.Pop();
                            indents.Push(indent);
                        } // if
                    } // if
                    else
                    {
                        indents.Push(indent);
                        tokens.Add(new Token(TokenType.BeginBlock, null));
                    } // else
                } // else
            } // if
            else if(indents.Peek() > indent)
            {
                while(indents.Count != 0 && indents.Peek() != indent)
                {
                    indents.Pop();
                    tokens.Add(new Token(TokenType.EndBlock, null));
                } // while
            } // else if
             * */
        }
예제 #7
0
        private static TokenSequence OptimizeTokens(TokenSequence tokens)
        {
            TokenSequence optimizedTokens = new TokenSequence();

            //
            // Removing sequences of Indent(EndStatement)+ tokens
            // and compacting EndStatement+ tokens to just one
            while(!tokens.Empty)
            {
                Token token = tokens.RemoveFirst();

                if(token.Type == TokenType.Indent && !tokens.Empty && tokens.First.Type == TokenType.EndStatement)
                    while(!tokens.Empty && tokens.First.Type == TokenType.EndStatement)
                        tokens.RemoveFirst();
                else if(token.Type == TokenType.EndStatement && !tokens.Empty && tokens.First.Type == TokenType.EndStatement)
                {
                    while(!tokens.Empty && tokens.First.Type == TokenType.EndStatement)
                        tokens.RemoveFirst();

                    optimizedTokens.Add(new Token(TokenType.EndStatement, null));
                } // else if
                else
                    optimizedTokens.Add(token);
            } // while

            return optimizedTokens;
        }
예제 #8
0
        private static TokenSequence LayoutTokens(TokenSequence tokens)
        {
            TokenSequence laidOutTokens = new TokenSequence();

            Stack<int> indents = new Stack<int>();
            indents.Push(0);

            while(!tokens.Empty)
            {
                Token token = tokens.RemoveFirst();

                if(token.Type == TokenType.Indent)
                {
                    if(token.Lexeme.Length == indents.Peek())
                        continue;

                    if(token.Lexeme.Length > indents.Peek())
                    {
                        indents.Push(token.Lexeme.Length);
                        laidOutTokens.Add(new Token(TokenType.BeginBlock, token.Location));
                    } // if
                    else
                    {
                        while(indents.Peek() != 0 && indents.Peek() != token.Lexeme.Length)
                        {
                            indents.Pop();
                            laidOutTokens.Add(new Token(TokenType.EndBlock, token.Location));
                        } // while
                    } // else
                } // if
                else 
                    laidOutTokens.Add(token);
            } // while

            //
            // Close remaining blocks
            while(indents.Peek() != 0)
            {
                indents.Pop();
                laidOutTokens.Add(new Token(TokenType.EndBlock, null));
            } // while

            return laidOutTokens;
        }
예제 #9
0
 private static TokenSequence PostProcessTokens(TokenSequence tokens)
 {
     return LayoutTokens(OptimizeTokens(tokens));
 }
        public void RemoveFirst()
        {
            TokenSequence sequence = new TokenSequence();
            sequence.Add(new Token(TokenType.BeginBlock, null));
            sequence.Add(new Token(TokenType.Colon, null));
            sequence.Add(new Token(TokenType.Comma, null));

            Assert.AreEqual(new Token(TokenType.BeginBlock, null), sequence.RemoveFirst());
            Assert.AreEqual(2, sequence.Count);
        }
 public void LastThrowsInvalidOperationExceptionOnEmptySequence()
 {
     TokenSequence sequence = new TokenSequence();
     Token last = sequence.Last;
 }