Example #1
0
 public void UpdateRange(Token token)
 {
     if (token.StartPos < startpos) startpos = token.StartPos;
     if (token.EndPos > endpos) endpos = token.EndPos;
 }
Example #2
0
        public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.WHITESPACE);

            regex = new Regex(@"^\s*$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EOF, regex);
            Tokens.Add(TokenType.EOF);

            regex = new Regex(@"(\[@?[A-Z]+[%]?\])(\.[A-Za-z]+)?", RegexOptions.Compiled);
            Patterns.Add(TokenType.VARIABLE, regex);
            Tokens.Add(TokenType.VARIABLE);

            //regex = new Regex(@"\""[0-9A-Za-z' :\-]*\""", RegexOptions.Compiled);
            // fix for all language (chinese,russian, etc.)
            regex = new Regex(@"\""([^\""]*)\""", RegexOptions.Compiled);
            Patterns.Add(TokenType.STRING, regex);
            Tokens.Add(TokenType.STRING);

            regex = new Regex(@"[0-9]*(\.[0-9]+)?", RegexOptions.Compiled);
            Patterns.Add(TokenType.NUMBER, regex);
            Tokens.Add(TokenType.NUMBER);

            regex = new Regex(@"[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]", RegexOptions.Compiled);
            Patterns.Add(TokenType.BOOLEAN, regex);
            Tokens.Add(TokenType.BOOLEAN);

            regex = new Regex(@"#", RegexOptions.Compiled);
            Patterns.Add(TokenType.SEPARATOR, regex);
            Tokens.Add(TokenType.SEPARATOR);

            regex = new Regex(@"\|\|", RegexOptions.Compiled);
            Patterns.Add(TokenType.OR, regex);
            Tokens.Add(TokenType.OR);

            regex = new Regex(@"&&", RegexOptions.Compiled);
            Patterns.Add(TokenType.AND, regex);
            Tokens.Add(TokenType.AND);

            regex = new Regex(@"==", RegexOptions.Compiled);
            Patterns.Add(TokenType.EQUAL, regex);
            Tokens.Add(TokenType.EQUAL);

            regex = new Regex(@"!=", RegexOptions.Compiled);
            Patterns.Add(TokenType.NOTEQUAL, regex);
            Tokens.Add(TokenType.NOTEQUAL);

            regex = new Regex(@"<=", RegexOptions.Compiled);
            Patterns.Add(TokenType.SMALLEQ, regex);
            Tokens.Add(TokenType.SMALLEQ);

            regex = new Regex(@">=", RegexOptions.Compiled);
            Patterns.Add(TokenType.GREATEQ, regex);
            Tokens.Add(TokenType.GREATEQ);

            regex = new Regex(@"<", RegexOptions.Compiled);
            Patterns.Add(TokenType.SMALLTH, regex);
            Tokens.Add(TokenType.SMALLTH);

            regex = new Regex(@">", RegexOptions.Compiled);
            Patterns.Add(TokenType.GREATTH, regex);
            Tokens.Add(TokenType.GREATTH);

            regex = new Regex(@"\+", RegexOptions.Compiled);
            Patterns.Add(TokenType.PLUS, regex);
            Tokens.Add(TokenType.PLUS);

            regex = new Regex(@"-", RegexOptions.Compiled);
            Patterns.Add(TokenType.MINUS, regex);
            Tokens.Add(TokenType.MINUS);

            regex = new Regex(@"\*", RegexOptions.Compiled);
            Patterns.Add(TokenType.MULT, regex);
            Tokens.Add(TokenType.MULT);

            regex = new Regex(@"\/", RegexOptions.Compiled);
            Patterns.Add(TokenType.DIV, regex);
            Tokens.Add(TokenType.DIV);

            regex = new Regex(@"\(", RegexOptions.Compiled);
            Patterns.Add(TokenType.BROPEN, regex);
            Tokens.Add(TokenType.BROPEN);

            regex = new Regex(@"\)", RegexOptions.Compiled);
            Patterns.Add(TokenType.BRCLOSE, regex);
            Tokens.Add(TokenType.BRCLOSE);

            regex = new Regex(@"\s+", RegexOptions.Compiled);
            Patterns.Add(TokenType.WHITESPACE, regex);
            Tokens.Add(TokenType.WHITESPACE);
        }
Example #3
0
 /// <summary>
 /// executes a lookahead of the next token
 /// and will advance the scan on the input string
 /// </summary>
 /// <returns></returns>
 public Token Scan(params TokenType[] expectedtokens)
 {
     Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
     LookAheadToken = null; // reset lookahead token, so scanning will continue
     StartPos = tok.EndPos;
     EndPos = tok.EndPos; // set the tokenizer to the new scan position
     return tok;
 }
Example #4
0
        /// <summary>
        /// returns token with longest best match
        /// </summary>
        /// <returns></returns>
        public Token LookAhead(params TokenType[] expectedtokens)
        {
            int i;
            int startpos = StartPos;
            Token tok = null;
            List<TokenType> scantokens;

            // this prevents double scanning and matching
            // increased performance
            if (LookAheadToken != null
                && LookAheadToken.Type != TokenType._UNDETERMINED_
                && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;

            // if no scantokens specified, then scan for all of them (= backward compatible)
            if (expectedtokens.Length == 0)
                scantokens = Tokens;
            else
            {
                scantokens = new List<TokenType>(expectedtokens);
                scantokens.AddRange(SkipList);
            }

            do
            {

                int len = -1;
                TokenType index = (TokenType)int.MaxValue;
                string input = Input.Substring(startpos);

                tok = new Token(startpos, EndPos);

                for (i = 0; i < scantokens.Count; i++)
                {
                    Regex r = Patterns[scantokens[i]];
                    Match m = r.Match(input);
                    if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len)))
                    {
                        len = m.Length;
                        index = scantokens[i];
                    }
                }

                if (index >= 0 && len >= 0)
                {
                    tok.EndPos = startpos + len;
                    tok.Text = Input.Substring(tok.StartPos, len);
                    tok.Type = index;
                }
                else if (tok.StartPos < tok.EndPos - 1)
                {
                    tok.Text = Input.Substring(tok.StartPos, 1);
                }

                if (SkipList.Contains(tok.Type))
                {
                    startpos = tok.EndPos;
                    Skipped.Add(tok);
                }
                else
                {
                    // only assign to non-skipped tokens
                    tok.Skipped = Skipped; // assign prior skips to this token
                    Skipped = new List<Token>(); //reset skips
                }
            }
            while (SkipList.Contains(tok.Type));

            LookAheadToken = tok;
            return tok;
        }
Example #5
0
 public void Init(string input)
 {
     this.Input = input;
     StartPos = 0;
     EndPos = 0;
     CurrentLine = 0;
     CurrentColumn = 0;
     CurrentPosition = 0;
     LookAheadToken = null;
 }
Example #6
0
 public Token GetToken(TokenType type)
 {
     Token t = new Token(this.StartPos, this.EndPos);
     t.Type = type;
     return t;
 }
Example #7
0
        /// <summary>
        /// inserts the RTF codes to highlight text blocks
        /// </summary>
        /// <param name="token">the token to highlight, will be appended to sb</param>
        /// <param name="sb">the final output string</param>
        private void HighlightToken(Token token, StringBuilder sb)
        {
            switch (token.Type)
            {
                case TokenType.VARIABLE:
                    sb.Append(@"{{\cf1 ");
                    break;
                case TokenType.STRING:
                    sb.Append(@"{{\cf2 ");
                    break;
                case TokenType.NUMBER:
                    sb.Append(@"{{\cf3 ");
                    break;
                case TokenType.BOOLEAN:
                    sb.Append(@"{{\cf4 ");
                    break;
                case TokenType.SEPARATOR:
                    sb.Append(@"{{\cf5 ");
                    break;
                case TokenType.OR:
                    sb.Append(@"{{\cf6 ");
                    break;
                case TokenType.AND:
                    sb.Append(@"{{\cf7 ");
                    break;
                case TokenType.EQUAL:
                    sb.Append(@"{{\cf8 ");
                    break;
                case TokenType.NOTEQUAL:
                    sb.Append(@"{{\cf9 ");
                    break;
                case TokenType.SMALLEQ:
                    sb.Append(@"{{\cf10 ");
                    break;
                case TokenType.GREATEQ:
                    sb.Append(@"{{\cf11 ");
                    break;
                case TokenType.SMALLTH:
                    sb.Append(@"{{\cf12 ");
                    break;
                case TokenType.GREATTH:
                    sb.Append(@"{{\cf13 ");
                    break;
                case TokenType.PLUS:
                    sb.Append(@"{{\cf14 ");
                    break;
                case TokenType.MINUS:
                    sb.Append(@"{{\cf15 ");
                    break;
                case TokenType.MULT:
                    sb.Append(@"{{\cf16 ");
                    break;
                case TokenType.DIV:
                    sb.Append(@"{{\cf17 ");
                    break;
                case TokenType.BROPEN:
                    sb.Append(@"{{\cf18 ");
                    break;
                case TokenType.BRCLOSE:
                    sb.Append(@"{{\cf19 ");
                    break;

                default:
                    sb.Append(@"{{\cf0 ");
                    break;
            }
        }