public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.WHITESPACE);

            regex = new Regex(@"[\s\t\r\n]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.WHITESPACE, regex);
            Tokens.Add(TokenType.WHITESPACE);

            regex = new Regex(@"[0-9]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.NUMBER, regex);
            Tokens.Add(TokenType.NUMBER);

            regex = new Regex(@"[a-zA-Z0-9_']+", RegexOptions.Compiled);
            Patterns.Add(TokenType.IDENTIFIER, regex);
            Tokens.Add(TokenType.IDENTIFIER);

            regex = new Regex(@"^$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EOF, regex);
            Tokens.Add(TokenType.EOF);

            regex = new Regex(@"parity", RegexOptions.Compiled);
            Patterns.Add(TokenType.PARITY, regex);
            Tokens.Add(TokenType.PARITY);

            regex = new Regex(@"[01]", RegexOptions.Compiled);
            Patterns.Add(TokenType.OWNERSTR, regex);
            Tokens.Add(TokenType.OWNERSTR);

            regex = new Regex(@",", RegexOptions.Compiled);
            Patterns.Add(TokenType.COMMA, regex);
            Tokens.Add(TokenType.COMMA);

            regex = new Regex(";", RegexOptions.Compiled);
            Patterns.Add(TokenType.SEMICOLON, regex);
            Tokens.Add(TokenType.SEMICOLON);

            regex = new Regex(@"""[^""]*""", RegexOptions.Compiled);
            Patterns.Add(TokenType.STRING, regex);
            Tokens.Add(TokenType.STRING);
        }
 public void UpdateRange(Token token)
 {
     if (token.StartPos < startpos) startpos = token.StartPos;
     if (token.EndPos > endpos) endpos = token.EndPos;
 }
 /// <summary>
 /// executes a lookahead of the next token
 /// and will advance the scan on the input string
 /// </summary>
 /// <returns></returns>
 public Token Scan(params TokenType[] expectedtokens)
 {
     Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
     LookAheadToken = null; // reset lookahead token, so scanning will continue
     StartPos = tok.EndPos;
     EndPos = tok.EndPos; // set the tokenizer to the new scan position
     return tok;
 }
        /// <summary>
        /// returns token with longest best match
        /// </summary>
        /// <returns></returns>
        public Token LookAhead(params TokenType[] expectedtokens)
        {
            int i;
            int startpos = StartPos;
            Token tok = null;
            List<TokenType> scantokens;

            // this prevents double scanning and matching
            // increased performance
            if (LookAheadToken != null
                && LookAheadToken.Type != TokenType._UNDETERMINED_
                && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;

            // if no scantokens specified, then scan for all of them (= backward compatible)
            if (expectedtokens.Length == 0)
                scantokens = Tokens;
            else {
                scantokens = new List<TokenType>(expectedtokens);
                scantokens.AddRange(SkipList);
            }

            do {

                int len = -1;
                TokenType index = (TokenType)int.MaxValue;
                string input = Input.Substring(startpos);

                tok = new Token(startpos, EndPos);

                for (i = 0; i < scantokens.Count; i++) {
                    Regex r = Patterns[scantokens[i]];
                    Match m = r.Match(input);
                    if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len))) {
                        len = m.Length;
                        index = scantokens[i];
                    }
                }

                if (index >= 0 && len >= 0) {
                    tok.EndPos = startpos + len;
                    tok.Text = Input.Substring(tok.StartPos, len);
                    tok.Type = index;
                }
                else if (tok.StartPos < tok.EndPos - 1) {
                    tok.Text = Input.Substring(tok.StartPos, 1);
                }

                if (SkipList.Contains(tok.Type)) {
                    startpos = tok.EndPos;
                    Skipped.Add(tok);
                }
                else {
                    // only assign to non-skipped tokens
                    tok.Skipped = Skipped; // assign prior skips to this token
                    Skipped = new List<Token>(); //reset skips
                }
            }
            while (SkipList.Contains(tok.Type));

            LookAheadToken = tok;
            return tok;
        }
 public void Init(string input)
 {
     this.Input = input;
     StartPos = 0;
     EndPos = 0;
     CurrentLine = 0;
     CurrentColumn = 0;
     CurrentPosition = 0;
     LookAheadToken = null;
 }
 public Token GetToken(TokenType type)
 {
     Token t = new Token(this.StartPos, this.EndPos);
     t.Type = type;
     return t;
 }