public Scanner() { Regex regex; Patterns = new Dictionary<TokenType, Regex>(); Tokens = new List<TokenType>(); LookAheadToken = null; Skipped = new List<Token>(); SkipList = new List<TokenType>(); SkipList.Add(TokenType.WHITESPACE); regex = new Regex(@"[0-9]+", RegexOptions.Compiled); Patterns.Add(TokenType.DECIMALINTEGERLITERAL, regex); Tokens.Add(TokenType.DECIMALINTEGERLITERAL); regex = new Regex(@"[a-zA-Z0-9_ ]*", RegexOptions.Compiled); Patterns.Add(TokenType.STRINGLITERAL, regex); Tokens.Add(TokenType.STRINGLITERAL); regex = new Regex(@"{\s*", RegexOptions.Compiled); Patterns.Add(TokenType.BRACEOPEN, regex); Tokens.Add(TokenType.BRACEOPEN); regex = new Regex(@"\s*}", RegexOptions.Compiled); Patterns.Add(TokenType.BRACECLOSE, regex); Tokens.Add(TokenType.BRACECLOSE); regex = new Regex(@"\(\s*", RegexOptions.Compiled); Patterns.Add(TokenType.BRACKETOPEN, regex); Tokens.Add(TokenType.BRACKETOPEN); regex = new Regex(@"\s*\)", RegexOptions.Compiled); Patterns.Add(TokenType.BRACKETCLOSE, regex); Tokens.Add(TokenType.BRACKETCLOSE); regex = new Regex(@"\|", RegexOptions.Compiled); Patterns.Add(TokenType.PIPE, regex); Tokens.Add(TokenType.PIPE); regex = new Regex(@"&", RegexOptions.Compiled); Patterns.Add(TokenType.AMP, regex); Tokens.Add(TokenType.AMP); regex = new Regex(@"!", RegexOptions.Compiled); Patterns.Add(TokenType.NOT, regex); Tokens.Add(TokenType.NOT); regex = new Regex(@",", RegexOptions.Compiled); Patterns.Add(TokenType.COMMA, regex); Tokens.Add(TokenType.COMMA); regex = new Regex(@"^$", RegexOptions.Compiled); Patterns.Add(TokenType.EOF, regex); Tokens.Add(TokenType.EOF); regex = new Regex(@"\s+", RegexOptions.Compiled); Patterns.Add(TokenType.WHITESPACE, regex); Tokens.Add(TokenType.WHITESPACE); }
public void UpdateRange(Token token) { if (token.StartPos < startpos) startpos = token.StartPos; if (token.EndPos > endpos) endpos = token.EndPos; }
/// <summary> /// executes a lookahead of the next token /// and will advance the scan on the input string /// </summary> /// <returns></returns> public Token Scan(params TokenType[] expectedtokens) { Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead LookAheadToken = null; // reset lookahead token, so scanning will continue StartPos = tok.EndPos; EndPos = tok.EndPos; // set the tokenizer to the new scan position return tok; }
/// <summary> /// returns token with longest best match /// </summary> /// <returns></returns> public Token LookAhead(params TokenType[] expectedtokens) { int i; int startpos = StartPos; Token tok = null; List<TokenType> scantokens; // this prevents double scanning and matching // increased performance if (LookAheadToken != null && LookAheadToken.Type != TokenType._UNDETERMINED_ && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken; // if no scantokens specified, then scan for all of them (= backward compatible) if (expectedtokens.Length == 0) scantokens = Tokens; else { scantokens = new List<TokenType>(expectedtokens); scantokens.AddRange(SkipList); } do { int len = -1; TokenType index = (TokenType)int.MaxValue; string input = Input.Substring(startpos); tok = new Token(startpos, EndPos); for (i = 0; i < scantokens.Count; i++) { Regex r = Patterns[scantokens[i]]; Match m = r.Match(input); if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len ))) { len = m.Length; index = scantokens[i]; } } if (index >= 0 && len >= 0) { tok.EndPos = startpos + len; tok.Text = Input.Substring(tok.StartPos, len); tok.Type = index; } else if (tok.StartPos < tok.EndPos - 1) { tok.Text = Input.Substring(tok.StartPos, 1); } if (SkipList.Contains(tok.Type)) { startpos = tok.EndPos; Skipped.Add(tok); } else { // only assign to non-skipped tokens tok.Skipped = Skipped; // assign prior skips to this token Skipped = new List<Token>(); //reset skips } } while (SkipList.Contains(tok.Type)); LookAheadToken = tok; return tok; }
public void Init(string input) { this.Input = input; StartPos = 0; EndPos = 0; CurrentLine = 0; CurrentColumn = 0; CurrentPosition = 0; LookAheadToken = null; }
public Token GetToken(TokenType type) { Token t = new Token(this.StartPos, this.EndPos); t.Type = type; return t; }
/// <summary> /// inserts the RTF codes to highlight text blocks /// </summary> /// <param name="token">the token to highlight, will be appended to sb</param> /// <param name="sb">the final output string</param> private void HighlightToken(Token token, StringBuilder sb) { switch (token.Type) { case TokenType.DECIMALINTEGERLITERAL: sb.Append(@"{{\cf1 "); break; case TokenType.STRINGLITERAL: sb.Append(@"{{\cf2 "); break; default: sb.Append(@"{{\cf0 "); break; } }