public void UpdateRange(Token token) { if (token.StartPos < startpos) startpos = token.StartPos; if (token.EndPos > endpos) endpos = token.EndPos; }
/// <summary> /// executes a lookahead of the next token /// and will advance the scan on the input string /// </summary> /// <returns></returns> public Token Scan(params TokenType[] expectedtokens) { Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead LookAheadToken = null; // reset lookahead token, so scanning will continue StartPos = tok.EndPos; EndPos = tok.EndPos; // set the tokenizer to the new scan position return tok; }
/// <summary> /// returns token with longest best match /// </summary> /// <returns></returns> public Token LookAhead(params TokenType[] expectedtokens) { int i; int startpos = StartPos; Token tok = null; List<TokenType> scantokens; // this prevents double scanning and matching // increased performance if (LookAheadToken != null && LookAheadToken.Type != TokenType._UNDETERMINED_ && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken; // if no scantokens specified, then scan for all of them (= backward compatible) if (expectedtokens.Length == 0) scantokens = Tokens; else { scantokens = new List<TokenType>(expectedtokens); scantokens.AddRange(SkipList); } do { int len = -1; TokenType index = (TokenType)int.MaxValue; string input = Input.Substring(startpos); tok = new Token(startpos, EndPos); for (i = 0; i < scantokens.Count; i++) { Regex r = Patterns[scantokens[i]]; Match m = r.Match(input); if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len ))) { len = m.Length; index = scantokens[i]; } } if (index >= 0 && len >= 0) { tok.EndPos = startpos + len; tok.Text = Input.Substring(tok.StartPos, len); tok.Type = index; } else if (tok.StartPos < tok.EndPos - 1) { tok.Text = Input.Substring(tok.StartPos, 1); } if (SkipList.Contains(tok.Type)) { startpos = tok.EndPos; Skipped.Add(tok); } else { // only assign to non-skipped tokens tok.Skipped = Skipped; // assign prior skips to this token Skipped = new List<Token>(); //reset skips } } while (SkipList.Contains(tok.Type)); LookAheadToken = tok; return tok; }
public void Init(string input) { this.Input = input; StartPos = 0; EndPos = 0; CurrentLine = 0; CurrentColumn = 0; CurrentPosition = 0; LookAheadToken = null; }
public Token GetToken(TokenType type) { Token t = new Token(this.StartPos, this.EndPos); t.Type = type; return t; }
public Scanner() { Regex regex; Patterns = new Dictionary<TokenType, Regex>(); Tokens = new List<TokenType>(); LookAheadToken = null; Skipped = new List<Token>(); SkipList = new List<TokenType>(); SkipList.Add(TokenType.WS); regex = new Regex(@"\+", RegexOptions.Compiled); Patterns.Add(TokenType.PLUS, regex); Tokens.Add(TokenType.PLUS); regex = new Regex(@"-", RegexOptions.Compiled); Patterns.Add(TokenType.MINUS, regex); Tokens.Add(TokenType.MINUS); regex = new Regex(@"\*", RegexOptions.Compiled); Patterns.Add(TokenType.STAR, regex); Tokens.Add(TokenType.STAR); regex = new Regex(@"\/", RegexOptions.Compiled); Patterns.Add(TokenType.FSLASH, regex); Tokens.Add(TokenType.FSLASH); regex = new Regex(@"\^", RegexOptions.Compiled); Patterns.Add(TokenType.EXPONENT, regex); Tokens.Add(TokenType.EXPONENT); regex = new Regex(@"\%", RegexOptions.Compiled); Patterns.Add(TokenType.MODULO, regex); Tokens.Add(TokenType.MODULO); regex = new Regex(@"\(", RegexOptions.Compiled); Patterns.Add(TokenType.LPAREN, regex); Tokens.Add(TokenType.LPAREN); regex = new Regex(@"\)", RegexOptions.Compiled); Patterns.Add(TokenType.RPAREN, regex); Tokens.Add(TokenType.RPAREN); regex = new Regex(@"\[", RegexOptions.Compiled); Patterns.Add(TokenType.LBRACKET, regex); Tokens.Add(TokenType.LBRACKET); regex = new Regex(@"\]", RegexOptions.Compiled); Patterns.Add(TokenType.RBRACKET, regex); Tokens.Add(TokenType.RBRACKET); regex = new Regex(@"\{", RegexOptions.Compiled); Patterns.Add(TokenType.LBRACE, regex); Tokens.Add(TokenType.LBRACE); regex = new Regex(@"\}", RegexOptions.Compiled); Patterns.Add(TokenType.RBRACE, regex); Tokens.Add(TokenType.RBRACE); regex = new Regex(@"=", RegexOptions.Compiled); Patterns.Add(TokenType.EQUAL, regex); Tokens.Add(TokenType.EQUAL); regex = new Regex(@"<", RegexOptions.Compiled); Patterns.Add(TokenType.LT, regex); Tokens.Add(TokenType.LT); regex = new Regex(@">", RegexOptions.Compiled); Patterns.Add(TokenType.GT, regex); Tokens.Add(TokenType.GT); regex = new Regex(@"<=", RegexOptions.Compiled); Patterns.Add(TokenType.LTE, regex); Tokens.Add(TokenType.LTE); regex = new Regex(@">=", RegexOptions.Compiled); Patterns.Add(TokenType.GTE, regex); Tokens.Add(TokenType.GTE); regex = new Regex(@"<>", RegexOptions.Compiled); Patterns.Add(TokenType.NOTEQUAL, regex); Tokens.Add(TokenType.NOTEQUAL); regex = new Regex(@"[Aa][Nn][Dd]", RegexOptions.Compiled); Patterns.Add(TokenType.AND, regex); Tokens.Add(TokenType.AND); regex = new Regex(@"[Oo][Rr]", RegexOptions.Compiled); Patterns.Add(TokenType.OR, regex); Tokens.Add(TokenType.OR); regex = new Regex(@"[Nn][Oo][Tt]", RegexOptions.Compiled); Patterns.Add(TokenType.NOT, regex); Tokens.Add(TokenType.NOT); regex = new Regex(@"\.", RegexOptions.Compiled); Patterns.Add(TokenType.DOT, regex); Tokens.Add(TokenType.DOT); regex = new Regex(@"\.\.", RegexOptions.Compiled); Patterns.Add(TokenType.DOTDOT, regex); Tokens.Add(TokenType.DOTDOT); regex = new Regex(@"\?", RegexOptions.Compiled); Patterns.Add(TokenType.QUIZ, regex); Tokens.Add(TokenType.QUIZ); regex = new Regex(@"\?:", RegexOptions.Compiled); Patterns.Add(TokenType.QUIZCOLON, regex); Tokens.Add(TokenType.QUIZCOLON); regex = new Regex(@",", RegexOptions.Compiled); Patterns.Add(TokenType.COMMA, regex); Tokens.Add(TokenType.COMMA); regex = new Regex(@":", RegexOptions.Compiled); Patterns.Add(TokenType.COLON, regex); Tokens.Add(TokenType.COLON); regex = new Regex(@"^$", RegexOptions.Compiled); Patterns.Add(TokenType.EOF, regex); Tokens.Add(TokenType.EOF); regex = new Regex(@"[Tt][Rr][Uu][Ee]", RegexOptions.Compiled); Patterns.Add(TokenType.TRUE, regex); Tokens.Add(TokenType.TRUE); regex = new Regex(@"[Ff][Aa][Ll][Ss][Ee]", RegexOptions.Compiled); Patterns.Add(TokenType.FALSE, regex); Tokens.Add(TokenType.FALSE); regex = new Regex(@"[Nn][Uu][Ll][Ll]", RegexOptions.Compiled); Patterns.Add(TokenType.NULL, regex); Tokens.Add(TokenType.NULL); regex = new Regex(@"(\+|-)?[0-9]+", RegexOptions.Compiled); Patterns.Add(TokenType.INTEGER, regex); Tokens.Add(TokenType.INTEGER); regex = new Regex(@"(\+|-)?[0-9]*\.[0-9]+", RegexOptions.Compiled); Patterns.Add(TokenType.DECIMAL, regex); Tokens.Add(TokenType.DECIMAL); regex = new Regex(@"[0-9]+\.\.[0-9]+", RegexOptions.Compiled); Patterns.Add(TokenType.INTRANGE, regex); Tokens.Add(TokenType.INTRANGE); regex = new Regex(@"""([^""\\]*(\\""[^""\\]*)*)""", RegexOptions.Compiled); Patterns.Add(TokenType.STRING, regex); Tokens.Add(TokenType.STRING); regex = new Regex(@"[a-zA-Z][a-zA-Z0-9_]*\(", RegexOptions.Compiled); Patterns.Add(TokenType.FUNCTION, regex); Tokens.Add(TokenType.FUNCTION); regex = new Regex(@"[a-zA-Z_$][a-zA-Z0-9_$]*", RegexOptions.Compiled); Patterns.Add(TokenType.IDENTIFIER, regex); Tokens.Add(TokenType.IDENTIFIER); regex = new Regex(@"\s+", RegexOptions.Compiled); Patterns.Add(TokenType.WS, regex); Tokens.Add(TokenType.WS); }