Beispiel #1
0
        public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.WHITESPACE);

            regex = new Regex(@"[0-9]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.DECIMALINTEGERLITERAL, regex);
            Tokens.Add(TokenType.DECIMALINTEGERLITERAL);

            regex = new Regex(@"[a-zA-Z0-9_ ]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.STRINGLITERAL, regex);
            Tokens.Add(TokenType.STRINGLITERAL);

            regex = new Regex(@"{\s*", RegexOptions.Compiled);
            Patterns.Add(TokenType.BRACEOPEN, regex);
            Tokens.Add(TokenType.BRACEOPEN);

            regex = new Regex(@"\s*}", RegexOptions.Compiled);
            Patterns.Add(TokenType.BRACECLOSE, regex);
            Tokens.Add(TokenType.BRACECLOSE);

            regex = new Regex(@"\(\s*", RegexOptions.Compiled);
            Patterns.Add(TokenType.BRACKETOPEN, regex);
            Tokens.Add(TokenType.BRACKETOPEN);

            regex = new Regex(@"\s*\)", RegexOptions.Compiled);
            Patterns.Add(TokenType.BRACKETCLOSE, regex);
            Tokens.Add(TokenType.BRACKETCLOSE);

            regex = new Regex(@"\|", RegexOptions.Compiled);
            Patterns.Add(TokenType.PIPE, regex);
            Tokens.Add(TokenType.PIPE);

            regex = new Regex(@"&", RegexOptions.Compiled);
            Patterns.Add(TokenType.AMP, regex);
            Tokens.Add(TokenType.AMP);

            regex = new Regex(@"!", RegexOptions.Compiled);
            Patterns.Add(TokenType.NOT, regex);
            Tokens.Add(TokenType.NOT);

            regex = new Regex(@",", RegexOptions.Compiled);
            Patterns.Add(TokenType.COMMA, regex);
            Tokens.Add(TokenType.COMMA);

            regex = new Regex(@"^$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EOF, regex);
            Tokens.Add(TokenType.EOF);

            regex = new Regex(@"\s+", RegexOptions.Compiled);
            Patterns.Add(TokenType.WHITESPACE, regex);
            Tokens.Add(TokenType.WHITESPACE);
        }
Beispiel #2
0
 public void UpdateRange(Token token)
 {
     if (token.StartPos < startpos) startpos = token.StartPos;
     if (token.EndPos > endpos) endpos = token.EndPos;
 }
Beispiel #3
0
 /// <summary>
 /// executes a lookahead of the next token
 /// and will advance the scan on the input string
 /// </summary>
 /// <returns></returns>
 public Token Scan(params TokenType[] expectedtokens)
 {
     Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
     LookAheadToken = null; // reset lookahead token, so scanning will continue
     StartPos = tok.EndPos;
     EndPos = tok.EndPos; // set the tokenizer to the new scan position
     return tok;
 }
Beispiel #4
0
        /// <summary>
        /// returns token with longest best match
        /// </summary>
        /// <returns></returns>
        public Token LookAhead(params TokenType[] expectedtokens)
        {
            int i;
            int startpos = StartPos;
            Token tok = null;
            List<TokenType> scantokens;

            // this prevents double scanning and matching
            // increased performance
            if (LookAheadToken != null
                && LookAheadToken.Type != TokenType._UNDETERMINED_
                && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;

            // if no scantokens specified, then scan for all of them (= backward compatible)
            if (expectedtokens.Length == 0)
                scantokens = Tokens;
            else
            {
                scantokens = new List<TokenType>(expectedtokens);
                scantokens.AddRange(SkipList);
            }

            do
            {

                int len = -1;
                TokenType index = (TokenType)int.MaxValue;
                string input = Input.Substring(startpos);

                tok = new Token(startpos, EndPos);

                for (i = 0; i < scantokens.Count; i++)
                {
                    Regex r = Patterns[scantokens[i]];
                    Match m = r.Match(input);
                    if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len )))
                    {
                        len = m.Length;
                        index = scantokens[i];
                    }
                }

                if (index >= 0 && len >= 0)
                {
                    tok.EndPos = startpos + len;
                    tok.Text = Input.Substring(tok.StartPos, len);
                    tok.Type = index;
                }
                else if (tok.StartPos < tok.EndPos - 1)
                {
                    tok.Text = Input.Substring(tok.StartPos, 1);
                }

                if (SkipList.Contains(tok.Type))
                {
                    startpos = tok.EndPos;
                    Skipped.Add(tok);
                }
                else
                {
                    // only assign to non-skipped tokens
                    tok.Skipped = Skipped; // assign prior skips to this token
                    Skipped = new List<Token>(); //reset skips
                }
            }
            while (SkipList.Contains(tok.Type));

            LookAheadToken = tok;
            return tok;
        }
Beispiel #5
0
 public void Init(string input)
 {
     this.Input = input;
     StartPos = 0;
     EndPos = 0;
     CurrentLine = 0;
     CurrentColumn = 0;
     CurrentPosition = 0;
     LookAheadToken = null;
 }
Beispiel #6
0
 public Token GetToken(TokenType type)
 {
     Token t = new Token(this.StartPos, this.EndPos);
     t.Type = type;
     return t;
 }
        /// <summary>
        /// inserts the RTF codes to highlight text blocks
        /// </summary>
        /// <param name="token">the token to highlight, will be appended to sb</param>
        /// <param name="sb">the final output string</param>
        private void HighlightToken(Token token, StringBuilder sb)
        {
            switch (token.Type)
            {
                    case TokenType.DECIMALINTEGERLITERAL:
                        sb.Append(@"{{\cf1 ");
                        break;
                    case TokenType.STRINGLITERAL:
                        sb.Append(@"{{\cf2 ");
                        break;

                default:
                    sb.Append(@"{{\cf0 ");
                    break;
            }
        }