//----< keep extracting until get none-alpha >-------------------

            override public Token getTok()
            {
                Token tok = new Token();

                tok.Append((char)context_.src.next());
                tok.Append((char)context_.src.next());// return two char
                return(tok);
            }
Example #2
0
        //----< keep extracting until get none-alpha >-------------------

        override public Token getTok()
        {
            Token tok = new Token();

            tok.Append((char)context_.src.next());          // first special char
            tok.Append((char)context_.src.next());          // second special char
            return(tok);
        }
Example #3
0
        //----< keep extracting until get none-in-quote >-------------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());
            while ((char)context_.src.peek() != '\r')
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #4
0
        //----< keep extracting until get none-whitespace >--------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());     // first is WhiteSpace

            while (isWhiteSpace(context_.src.peek()))  // stop when non-WhiteSpace
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #5
0
        //----< keep extracting until get none-alpha >-------------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());          

            while (isLetterOrDigit(context_.src.peek()))    
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #6
0
        //----< keep extracting until get none-punctuator >--------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());

            while (isPunctuation(context_.src.peek()))
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #7
0
        //----< manage converting extracted ints to chars and doing comment check>--------------

        public override Token getTok()
        {
            Token tok = new Token();

            tok.Append((char)context_.src.next());                      // first is backslash
            while (context_.src.peek() != 13)
            {
                tok.Append((char)context_.src.next());                  // add everything untill newline is reached
            }
            return(tok);
        }
Example #8
0
        //----< keep extracting until get none-alpha >-------------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());          // first is alpha

            while (isLetterOrDigit(context_.src.peek())||context_.src.peek()=='_')    // stop when non-alpha
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #9
0
        //----< keep extracting until get none-in-quote >-------------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());        
               
            while (!isDoubleQuote((char)context_.src.peek()))
            {
                tok.Append((char)context_.src.next());
            }
            tok.Append((char)context_.src.next());
            return tok;
        }
Example #10
0
        //----< keep extracting until get none-in-quote >-------------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());
            while (((char)context_.src.peek() != '/'))
            {
                tok.Append((char)context_.src.next());
            }
            tok.Append((char)context_.src.next());
            tok = tok.Replace(Environment.NewLine, " ");
            return tok;
        }
Example #11
0
        //----< keep extracting until get none-punctuator >--------------

        override public Token getTok()
        {
            Token tok = new Token();

            tok.Append((char)context_.src.next());       // first is punctuator

            while (isPunctuation(context_.src.peek()))   // stop when non-punctuator
            {
                tok.Append((char)context_.src.next());
            }
            return(tok);
        }
Example #12
0
        //----< keep extracting until get none-NewLine >--------------

        override public Token getTok()
        {
            Token tok = new Token();

            //tok.Append("\\n");
            //context_.src.next();
            tok.Append((char)context_.src.next());  // first is NewLine
            while (isNewLine(context_.src.peek()))  // stop when non-NewLine
            {
                tok.Append((char)context_.src.next());
                //tok.Append("\\n");
            }
            return(tok);
        }
Example #13
0
        //----< keep extracting until get non-quote >-------------------

        override public Token getTok()
        {
            Token tok = new Token();

            tok.Append((char)context_.src.next());       // first is quotes

            while ((char)context_.src.peek() != '\'')    // stop when quotes again
            {
                tok.Append((char)context_.src.next());
                if (tok.Length > 1 && escape(tok))
                {
                    tok.Append((char)context_.src.next());
                }
            }
            return(tok.Append((char)context_.src.next()));
        }
Example #14
0
        private static Token MakeToken(string text, int posIncr)
        {
            Token t = new Token();

            t.Append(text);
            t.PositionIncrement = posIncr;
            return(t);
        }
        public static StringBuilder Test_inter(String path)
        {
            Token   s  = new Token();
            SemiExp se = new SemiExp();

            s.Append("\n   - Semiexpression : " + se.getSemi(path)).Append(' ', 40 - se.getSemi(path).Length).Append("One SemiExpression.");
            return(s);
        }
        private Token MakeToken(string text, int posIncr, int startOffset, int endOffset)
        {
            Token t = new Token();

            t.Append(text);
            t.PositionIncrement = posIncr;
            t.SetOffset(startOffset, endOffset);
            return(t);
        }
Example #17
0
            //----< keep extracting until get none-alpha >-------------------

            override public Token getTok()
            {
                Token tok = new Token();

                tok.Append((char)context_.src.next());          // first is "/"
                bool endline = false;

                while (!endline)    // stop when the line end
                {
                    char ch = (char)context_.src.next();
                    tok.Append(ch);
                    if (ch == '\n')//return the entire line
                    {
                        endline = true;
                    }
                }
                return(tok);
            }
Example #18
0
        private LexerToken ScanHexadecimalString(EpsStreamReader reader)
        {
            Debug.Assert(reader.CurrentChar == Chars.Less);

            ClearToken();
            char[] hex = new char[2];
            ScanNextChar(reader);

            while (true)
            {
                MoveToNonWhiteSpace(reader);

                if (reader.CurrentChar == '>')
                {
                    ScanNextChar(reader);
                    break;
                }

                if (char.IsLetterOrDigit(reader.CurrentChar))
                {
                    hex[0] = char.ToUpper(reader.CurrentChar);

                    var nextChar = reader.NextChar;

                    if (nextChar != '>')
                    {
                        hex[1] = char.ToUpper(nextChar);
                    }
                    else
                    {
                        hex[1] = '0';
                    }

                    int ch = int.Parse(new string(hex), NumberStyles.AllowHexSpecifier);
                    Token.Append(Convert.ToChar(ch));
                    ScanNextChar(reader);
                    ScanNextChar(reader);
                }
            }

            string chars = Token.ToString();
            int    count = chars.Length;

            if (count > 2 && chars[0] == (char)0xFE && chars[1] == (char)0xFF)
            {
                Debug.Assert(count % 2 == 0);
                Token.Length = 0;

                for (int idx = 2; idx < count; idx += 2)
                {
                    Token.Append((char)(chars[idx] * 256 + chars[idx + 1]));
                }
            }

            return(Symbol = LexerToken.HexString);
        }
Example #19
0
    public void AppendNull()
    {
        var text  = "something";
        var token = new Token(0, 0, 0, text.Length - 1, TokenType.Word, text);

        var newToken = token.Append(null);

        Assert.NotNull(newToken);
        Assert.Equal("something", newToken.Text);
    }
Example #20
0
        override public Token getTok()
        {
            Token tok = new Token();

            if (isSinglePunct(context_.src.peek()))
            {
                tok.Append((char)context_.src.next());
            }
            return tok;
        }
Example #21
0
            //----< keep extracting until get none-alpha >-------------------

            override public Token getTok()
            {
                Token tok = new Token();

                tok.Append((char)context_.src.next());          // first is '/'
                bool endcomment = false;
                char pre_ch     = '/';

                while (!endcomment)    // stop when comment is ended
                {
                    char cur_ch = (char)context_.src.next();
                    tok.Append(cur_ch);
                    if (cur_ch == '/' && pre_ch == '*')//end when "*/“ occurs
                    {
                        endcomment = true;
                    }
                    pre_ch = cur_ch;
                }
                return(tok);
            }
Example #22
0
    public void Append()
    {
        var text   = "something";
        var token  = new Token(0, 0, 0, text.Length - 1, TokenType.Word, text);
        var text2  = " other";
        var token2 = new Token(0, 0, 0, text2.Length - 1, TokenType.Word, text2);

        var newToken = token.Append(token2);

        Assert.NotNull(newToken);
        Assert.Equal("something other", newToken.Text);
        Assert.Equal(13, newToken.WordEnd);
    }
        public static StringBuilder Test_semi(String path)
        {
            StringBuilder         str   = new StringBuilder();
            SemiExp               s     = new SemiExp();
            int                   count = 0;
            List <List <string> > s_ex  = s.WholeSemi(path);

            foreach (List <String> t in s_ex)
            {
                if (count > 7)
                {
                    continue;
                }
                StringBuilder msg = new Token();
                for (int i = 0; i < t.Count(); i++)
                {
                    if (!t[i].Equals("\n"))
                    {
                        msg.Append(t[i]);
                        msg.Append(" ");
                    }
                }
                count++;
                str.Append("\n   -  " + msg).Append(' ', 60 - msg.Length);
                if (msg.ToString().Contains("#"))
                {
                    str.Append("| # condition satisfied");
                }
                if (msg.ToString().Contains("for"))
                {
                    str.Append("| for condition satisfied");
                }
                if (msg.ToString().Contains("{") || msg.ToString().Contains("}") || msg.ToString().Contains(";") && !msg.ToString().Contains("for"))
                {
                    str.Append("| end condition satisfied");
                }
            }
            return(str);
        }
        public static StringBuilder Test_semi_here(String path)
        {
            StringBuilder         str   = new StringBuilder();
            SemiExp               s     = new SemiExp();
            int                   count = 0;
            List <List <string> > s_ex  = s.WholeSemi(path);

            foreach (List <String> t in s_ex)
            {
                StringBuilder msg = new Token();
                for (int i = 0; i < t.Count(); i++)
                {
                    if (!t[i].Equals("\n"))
                    {
                        msg.Append(t[i]);
                        msg.Append(" ");
                    }
                }
                count++;
                str.Append("\n   -  " + msg);
            }
            return(str);
        }
Example #25
0
    // Run the integrated IPP code
    // @todo ipp: How to log the output in a nice UAT way?
    protected static int RunIPP(string IPPArguments)
    {
        List <string> Args = new List <string>();

        StringBuilder Token    = null;
        int           Index    = 0;
        bool          bInQuote = false;

        while (Index < IPPArguments.Length)
        {
            if (IPPArguments[Index] == '\"')
            {
                bInQuote = !bInQuote;
            }
            else if (IPPArguments[Index] == ' ' && !bInQuote)
            {
                if (Token != null)
                {
                    Args.Add(Token.ToString());
                    Token = null;
                }
            }
            else
            {
                if (Token == null)
                {
                    Token = new StringBuilder();
                }
                Token.Append(IPPArguments[Index]);
            }
            Index++;
        }

        if (Token != null)
        {
            Args.Add(Token.ToString());
        }

        return(RunIPP(Args.ToArray()));
    }
Example #26
0
		public Token Parse( Token source )
		{
			this.Source = source.String;
			int sourceEned = this.Source.Length;
			this.Line = 1;
			this.BOL = true;
			this.EnableOutput = true;

			//Accumulate ouput into this token
			var output = new Token( Token.Kind.Text );
			int emptyLines = 0;

			//Enabel output only if all embedded #if's were true
			bool oldOutputEnabled = true;
			bool outputEnabled = true;
			int outputDisabledLine = 0;

			for ( int i = 0; i < sourceEned; i++ )
			{
				int oldLine = this.Line;
				Token t = this.GetToken( true );

				NextToken:
				switch ( t.Type )
				{
					case Token.Kind.Error:
						return t;

					case Token.Kind.EOS:
						return output; //Force termination

					case Token.Kind.Comment:
						//C comments are replaced with single spaces.
						if ( outputEnabled )
						{
							output.Append( " ", 1 );
							output.AppendNL( this.Line - oldLine );
						}
						break;

					case Token.Kind.Linecomment:
						//C++ comments are ignored
						continue;
					case Token.Kind.Directive:
						t = this.HandleDirective( t, oldLine );
						outputEnabled = this.EnableOutput;
						if ( outputEnabled != oldOutputEnabled )
						{
							if ( outputEnabled )
							{
								output.AppendNL( oldLine - outputDisabledLine );
							}
							else
							{
								outputDisabledLine = oldLine;
							}
							oldOutputEnabled = outputEnabled;
						}

						if ( outputEnabled )
						{
							output.AppendNL( this.Line - oldLine - t.CountNL );
						}
						goto NextToken;

					case Token.Kind.Linecont:
						//Backslash-Newline sequences are delted, no matter where.
						emptyLines++;
						break;

					case Token.Kind.Newline:
						if ( emptyLines > 0 )
						{
							// Compensate for the backslash-newline combinations
							// we have encountered, otherwise line numeration is broken
							if ( outputEnabled )
							{
								output.AppendNL( emptyLines );
							}
							emptyLines = 0;
						}
						goto default;
					case Token.Kind.Whitespace:
					default:
						//Passthrough all other tokens
						if ( outputEnabled )
						{
							output.Append( t );
						}
						break;
				}
			}

			if ( this.EnableOutput == false )
			{
				this.Error( this.Line, "Unclosed #if at end of source", null );
				return Token.Error;
			}

			return output;
		}
Example #27
0
        /**
         * Implementation of the <i>Shunting Yard</i> algorithm to transform an
         * infix expression to a RPN expression.
         */
        internal List <Token> ShuntingYard(string expression)
        {
            List <Token>  outputQueue = new List <Token>();
            Stack <Token> stack       = new Stack <Token>();

            Tokenizer tokenizer = new Tokenizer(this, expression);

            Token lastFunction  = null;
            Token previousToken = null;

            foreach (Token token in tokenizer)
            {
                switch (token.type)
                {
                case TokenType.STRINGPARAM:
                    //stack.Push(token);
                    outputQueue.Add(token);
                    break;

                case TokenType.LITERAL:
                case TokenType.HEX_LITERAL:
                case TokenType.VARIABLE:
                    if (previousToken != null)
                    {
                        if (previousToken.type == TokenType.LITERAL || previousToken.type == TokenType.CLOSE_PAREN ||
                            previousToken.type == TokenType.VARIABLE ||
                            previousToken.type == TokenType.HEX_LITERAL)
                        {
                            // Implicit multiplication, e.g. 23(a+b) or (a+b)(a-b)
                            Token multiplication = new Token();
                            multiplication.Append("*");
                            multiplication.type = TokenType.OPERATOR;
                            stack.Push(multiplication);
                        }
                    }
                    outputQueue.Add(token);
                    break;

                case TokenType.OBJPATH:
                    outputQueue.Add(token);
                    break;

                case TokenType.FUNCTION:
                    stack.Push(token);
                    lastFunction = token;
                    break;

                case TokenType.COMMA:
                    if (previousToken != null && previousToken.type == TokenType.OPERATOR)
                    {
                        throw new ExpressionException("Missing parameter(s) for operator " + previousToken +
                                                      " at character position " + previousToken.pos);
                    }
                    while (!stack.IsEmpty() && stack.Peek().type != TokenType.OPEN_PAREN)
                    {
                        outputQueue.Add(stack.Pop());
                    }
                    if (stack.IsEmpty())
                    {
                        throw new ExpressionException("Parse error for function '"
                                                      + lastFunction + "'");
                    }
                    break;

                case TokenType.OPERATOR:
                {
                    if (previousToken != null && (previousToken.type == TokenType.COMMA || previousToken.type == TokenType.OPEN_PAREN))
                    {
                        throw new ExpressionException("Missing parameter(s) for operator " + token +
                                                      " at character position " + token.pos);
                    }
                    if (!operators.TryGetValue(token.surface, out Operator o1))
                    {
                        throw new ExpressionException("Unknown operator '" + token
                                                      + "' at position " + (token.pos + 1));
                    }

                    ShuntOperators(outputQueue, stack, o1);
                    stack.Push(token);
                    break;
                }

                case TokenType.UNARY_OPERATOR:
                {
                    if (previousToken != null && previousToken.type != TokenType.OPERATOR && previousToken.type != TokenType.UNARY_OPERATOR &&
                        previousToken.type != TokenType.COMMA && previousToken.type != TokenType.OPEN_PAREN)
                    {
                        throw new ExpressionException("Invalid position for unary operator " + token +
                                                      " at character position " + token.pos);
                    }
                    Operator o1;
                    if (operators.ContainsKey(token.surface))
                    {
                        o1 = operators[token.surface];
                    }
                    else
                    {
                        throw new ExpressionException("Unknown unary operator '"
                                                      + token.surface.Substring(0, token.surface.Length - 1)
                                                      + "' at position " + (token.pos + 1));
                    }

                    ShuntOperators(outputQueue, stack, o1);
                    stack.Push(token);
                    break;
                }

                case TokenType.OPEN_PAREN:
                    if (previousToken != null)
                    {
                        if (previousToken.type == TokenType.LITERAL || previousToken.type == TokenType.CLOSE_PAREN ||
                            previousToken.type == TokenType.VARIABLE ||
                            previousToken.type == TokenType.HEX_LITERAL)
                        {
                            // Implicit multiplication, e.g. 23(a+b) or (a+b)(a-b)
                            Token multiplication = new Token();
                            multiplication.Append("*");
                            multiplication.type = TokenType.OPERATOR;
                            stack.Push(multiplication);
                        }
                        // if the ( is preceded by a valid function, then it
                        // denotes the start of a parameter list
                        if (previousToken.type == TokenType.FUNCTION)
                        {
                            outputQueue.Add(token);
                        }
                    }
                    stack.Push(token);
                    break;

                case TokenType.CLOSE_PAREN:
                    if (previousToken != null && previousToken.type == TokenType.OPERATOR)
                    {
                        throw new ExpressionException("Missing parameter(s) for operator " + previousToken +
                                                      " at character position " + previousToken.pos);
                    }
                    if (previousToken != null && previousToken.type == TokenType.COMMA)
                    {
                        throw new ExpressionException("Missing parameter after comma at character position " + previousToken.pos);
                    }
                    while (!stack.IsEmpty() && stack.Peek().type != TokenType.OPEN_PAREN)
                    {
                        outputQueue.Add(stack.Pop());
                    }
                    if (stack.IsEmpty())
                    {
                        throw new ExpressionException("Mismatched parentheses");
                    }
                    stack.Pop();
                    if (!stack.IsEmpty() && stack.Peek().type == TokenType.FUNCTION)
                    {
                        outputQueue.Add(stack.Pop());
                    }
                    break;
                }
                previousToken = token;
            }

            while (!stack.IsEmpty())
            {
                Token element = stack.Pop();
                if (element.type == TokenType.OPEN_PAREN || element.type == TokenType.CLOSE_PAREN)
                {
                    throw new ExpressionException("Mismatched parentheses");
                }
                outputQueue.Add(element);
            }
            return(outputQueue);
        }
Example #28
0
 /// <summary>
 /// Appends current character to the token and reads next one.
 /// </summary>
 protected char AppendAndScanNextChar(EpsStreamReader reader)
 {
     Token.Append(reader.CurrentChar);
     return(ScanNextChar(reader));
 }
Example #29
0
 private Token MakeToken(string text, int posIncr, int startOffset, int endOffset)
 {
     Token t = new Token();
     t.Append(text);
     t.PositionIncrement = posIncr;
     t.SetOffset(startOffset, endOffset);
     return t;
 }
 private static Token MakeToken(string text, int posIncr)
 {
     Token t = new Token();
     t.Append(text);
     t.PositionIncrement = posIncr;
     return t;
 }
Example #31
0
            //----< keep extracting until get none-alpha >-------------------

            override public Token getTok()
            {
                Token tok   = new Token();
                char  quote = (char)context_.src.next();

                tok.Append(quote);          // first is \" or \'
                bool endquote = false;

                if (quote == '\'')
                {
                    int  count  = 0;
                    char pre_ch = '\'';
                    while (!endquote)    // stop when the quote complete
                    {
                        char cur_ch = (char)context_.src.next();
                        tok.Append(cur_ch);
                        if (cur_ch == '\\')
                        {
                            count++;
                        }
                        if (cur_ch == '\'')
                        {
                            if (pre_ch != '\\')
                            {
                                endquote = true;
                            }
                            if (pre_ch == '\\' && (count % 2 == 0)) //odd: '\'', even: '\\'
                            {
                                endquote = true;
                            }
                        }
                        pre_ch = cur_ch;
                    }
                    return(tok);
                }
                else
                {
                    int  count  = 0;
                    char pre_ch = '\"';
                    while (!endquote)    // stop when the quote complete
                    {
                        char cur_ch = (char)context_.src.next();
                        tok.Append(cur_ch);
                        if (cur_ch == '\\')
                        {
                            count++;
                        }
                        if (cur_ch == '\"')
                        {
                            if (pre_ch != '\\')
                            {
                                endquote = true;
                            }
                            if (pre_ch == '\\' && (count % 2 == 0)) //odd: "\"", even: "\\"
                            {
                                endquote = true;
                            }
                        }
                        pre_ch = cur_ch;
                    }
                    return(tok);
                }
            }
Example #32
0
        //----< keep extracting until get none-punctuator >--------------

        override public Token getTok()
        {
            Token tok = new Token();
            tok.Append((char)context_.src.next());       // first is punctuator

            if (tok[0] == '\"')
            {
                while (context_.src.peek() != '\"')
                {
                    tok.Append((char)context_.src.next());
                }
                tok.Append((char)context_.src.next());
            }

            else if(tok[0]=='\'')
            {
                while (context_.src.peek() != '\'')
                {
                    tok.Append((char)context_.src.next());
                }
                tok.Append((char)context_.src.next());
            }
            
            else if (tok[0] == '/')
            {
                if (context_.src.peek() == '/')     //single line comment
                {
                    while (context_.src.peek() != '\n')
                    {
                        tok.Append((char)context_.src.next());
                    }
                }

                else if (context_.src.peek() == '*')    //Multi-line comments
                {
                    tok.Append((char)context_.src.next());
                    while(!(tok[tok.Length-1]=='*' && context_.src.peek()=='/'))
                    {
                        tok.Append((char)context_.src.next());
                    }
                    tok.Append((char)context_.src.next());
                }

                else if (context_.src.peek() == '=')
                {
                    tok.Append((char)context_.src.next());
                }
            }

            //  special two pairs
            else if (tok[0] == '<')
            {
                if (context_.src.peek() == '<')
                {
                    tok.Append((char)context_.src.next());
                }
            }

            else if (tok[0] == '>')
            {
                if (context_.src.peek() == '>')
                {
                    tok.Append((char)context_.src.next());
                }
            }

            else if (tok[0] == ':')
            {
                if (context_.src.peek() == ':')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '=')
            {
                if (context_.src.peek() == '=')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '*')
            {
                if (context_.src.peek() == '=')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '|')
            {
                if (context_.src.peek() == '|')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '&')
            {
                if (context_.src.peek() == '&')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '+')
            {
                if (context_.src.peek() == '+'|| context_.src.peek() == '=')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            else if (tok[0] == '-')
            {
                if (context_.src.peek() == '-' || context_.src.peek() == '=')
                {
                    tok.Append((char)context_.src.next());
                }
            }
            return tok;
        }
Example #33
0
 private bool CopyChar()
 {
     m_token.Append(m_input.Char);
     return(m_input.Read());
 }