private List<TokenType> SkipList; // tokens to be skipped

        public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.WHITESPACE);

            regex = new Regex(@"[0-9]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.NUMBER, regex);
            Tokens.Add(TokenType.NUMBER);

            regex = new Regex(@"^$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EOF, regex);
            Tokens.Add(TokenType.EOF);

            regex = new Regex(@"'[^']+'", RegexOptions.Compiled);
            Patterns.Add(TokenType.STRING_PARAM, regex);
            Tokens.Add(TokenType.STRING_PARAM);

            regex = new Regex(@"\s+", RegexOptions.Compiled);
            Patterns.Add(TokenType.WHITESPACE, regex);
            Tokens.Add(TokenType.WHITESPACE);

            regex = new Regex(@"when", RegexOptions.Compiled);
            Patterns.Add(TokenType.WhenKeyword, regex);
            Tokens.Add(TokenType.WhenKeyword);

            regex = new Regex(@"subject contains", RegexOptions.Compiled);
            Patterns.Add(TokenType.SubjectContainsKeyword, regex);
            Tokens.Add(TokenType.SubjectContainsKeyword);

            regex = new Regex(@"then", RegexOptions.Compiled);
            Patterns.Add(TokenType.ThenKeyword, regex);
            Tokens.Add(TokenType.ThenKeyword);

            regex = new Regex(@"attach to project", RegexOptions.Compiled);
            Patterns.Add(TokenType.AttachToProjectKeyword, regex);
            Tokens.Add(TokenType.AttachToProjectKeyword);

            regex = new Regex(@"create request in project", RegexOptions.Compiled);
            Patterns.Add(TokenType.CreateRequestKeyword, regex);
            Tokens.Add(TokenType.CreateRequestKeyword);

            regex = new Regex(@"company matched to project", RegexOptions.Compiled);
            Patterns.Add(TokenType.CompanyMatchedKeyword, regex);
            Tokens.Add(TokenType.CompanyMatchedKeyword);

            regex = new Regex(@"and", RegexOptions.Compiled);
            Patterns.Add(TokenType.AndKeyword, regex);
            Tokens.Add(TokenType.AndKeyword);


        }
 public void Init(string input)
 {
     this.Input = input;
     StartPos = 0;
     EndPos = 0;
     CurrentLine = 0;
     CurrentColumn = 0;
     CurrentPosition = 0;
     LookAheadToken = null;
 }
 public void UpdateRange(Token token)
 {
     if (token.StartPos < startpos) startpos = token.StartPos;
     if (token.EndPos > endpos) endpos = token.EndPos;
 }
        /// <summary>
        /// returns token with longest best match
        /// </summary>
        /// <returns></returns>
        public Token LookAhead(params TokenType[] expectedtokens)
        {
            int i;
            int startpos = StartPos;
            Token tok = null;
            List<TokenType> scantokens;


            // this prevents double scanning and matching
            // increased performance
            if (LookAheadToken != null 
                && LookAheadToken.Type != TokenType._UNDETERMINED_ 
                && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;

            // if no scantokens specified, then scan for all of them (= backward compatible)
            if (expectedtokens.Length == 0)
                scantokens = Tokens;
            else
            {
                scantokens = new List<TokenType>(expectedtokens);
                scantokens.AddRange(SkipList);
            }

            do
            {

                int len = -1;
                TokenType index = (TokenType)int.MaxValue;
                string input = Input.Substring(startpos);

                tok = new Token(startpos, EndPos);

                for (i = 0; i < scantokens.Count; i++)
                {
                    Regex r = Patterns[scantokens[i]];
                    Match m = r.Match(input);
                    if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len )))
                    {
                        len = m.Length;
                        index = scantokens[i];  
                    }
                }

                if (index >= 0 && len >= 0)
                {
                    tok.EndPos = startpos + len;
                    tok.Text = Input.Substring(tok.StartPos, len);
                    tok.Type = index;
                }
                else if (tok.StartPos < tok.EndPos - 1)
                {
                    tok.Text = Input.Substring(tok.StartPos, 1);
                }

                if (SkipList.Contains(tok.Type))
                {
                    startpos = tok.EndPos;
                    Skipped.Add(tok);
                }
                else
                {
                    // only assign to non-skipped tokens
                    tok.Skipped = Skipped; // assign prior skips to this token
                    Skipped = new List<Token>(); //reset skips
                }
            }
            while (SkipList.Contains(tok.Type));

            LookAheadToken = tok;
            return tok;
        }
  /// <summary>
 /// executes a lookahead of the next token
 /// and will advance the scan on the input string
 /// </summary>
 /// <returns></returns>
 public Token Scan(params TokenType[] expectedtokens)
 {
     Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
     LookAheadToken = null; // reset lookahead token, so scanning will continue
     StartPos = tok.EndPos;
     EndPos = tok.EndPos; // set the tokenizer to the new scan position
     return tok;
 }
 public Token GetToken(TokenType type)
 {
     Token t = new Token(this.StartPos, this.EndPos);
     t.Type = type;
     return t;
 }