/// <summary> /// required to override this function from base otherwise the parsetree will consist of incorrect types of nodes /// </summary> /// <param name="token"></param> /// <param name="text"></param> /// <returns></returns> public override ParseNode CreateNode(Token token, string text) { ParseTreeEvaluator node = new ParseTreeEvaluator(); node.Token = token; node.text = text; node.Parent = this; return node; }
public void UpdateRange(Token token) { if (token.StartPos < startpos) startpos = token.StartPos; if (token.EndPos > endpos) endpos = token.EndPos; }
/// <summary> /// executes a lookahead of the next token /// and will advance the scan on the input string /// </summary> /// <returns></returns> public Token Scan(params TokenType[] expectedtokens) { Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead LookAheadToken = null; // reset lookahead token, so scanning will continue StartPos = tok.EndPos; EndPos = tok.EndPos; // set the tokenizer to the new scan position return tok; }
/// <summary> /// returns token with longest best match /// </summary> /// <returns></returns> public Token LookAhead(params TokenType[] expectedtokens) { int i; int startpos = StartPos; Token tok = null; List<TokenType> scantokens; // this prevents double scanning and matching // increased performance if (LookAheadToken != null && LookAheadToken.Type != TokenType._UNDETERMINED_ && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken; // if no scantokens specified, then scan for all of them (= backward compatible) if (expectedtokens.Length == 0) scantokens = Tokens; else { scantokens = new List<TokenType>(expectedtokens); scantokens.AddRange(SkipList); } do { int len = -1; TokenType index = (TokenType)int.MaxValue; string input = Input.Substring(startpos); tok = new Token(startpos, EndPos); for (i = 0; i < scantokens.Count; i++) { Regex r = Patterns[scantokens[i]]; Match m = r.Match(input); if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len ))) { len = m.Length; index = scantokens[i]; } } if (index >= 0 && len >= 0) { tok.EndPos = startpos + len; tok.Text = Input.Substring(tok.StartPos, len); tok.Type = index; } else if (tok.StartPos < tok.EndPos - 1) { tok.Text = Input.Substring(tok.StartPos, 1); } if (SkipList.Contains(tok.Type)) { startpos = tok.EndPos; Skipped.Add(tok); } else { // only assign to non-skipped tokens tok.Skipped = Skipped; // assign prior skips to this token Skipped = new List<Token>(); //reset skips } } while (SkipList.Contains(tok.Type)); LookAheadToken = tok; return tok; }
public void Init(string input) { this.Input = input; StartPos = 0; EndPos = 0; CurrentLine = 0; CurrentColumn = 0; CurrentPosition = 0; LookAheadToken = null; }
public Token GetToken(TokenType type) { Token t = new Token(this.StartPos, this.EndPos); t.Type = type; return t; }
public Scanner() { Regex regex; Patterns = new Dictionary<TokenType, Regex>(); Tokens = new List<TokenType>(); LookAheadToken = null; Skipped = new List<Token>(); SkipList = new List<TokenType>(); SkipList.Add(TokenType.WHITESPACE); regex = new Regex(@"true|false", RegexOptions.Compiled); Patterns.Add(TokenType.BOOLEANLITERAL, regex); Tokens.Add(TokenType.BOOLEANLITERAL); regex = new Regex(@"[0-9]+(UL|Ul|uL|ul|LU|Lu|lU|lu|U|u|L|l)?", RegexOptions.Compiled); Patterns.Add(TokenType.DECIMALINTEGERLITERAL, regex); Tokens.Add(TokenType.DECIMALINTEGERLITERAL); regex = new Regex(@"([0-9]+\.[0-9]+([eE][+-]?[0-9]+)?([fFdDMm]?)?)|(\.[0-9]+([eE][+-]?[0-9]+)?([fFdDMm]?)?)|([0-9]+([eE][+-]?[0-9]+)([fFdDMm]?)?)|([0-9]+([fFdDMm]?))", RegexOptions.Compiled); Patterns.Add(TokenType.REALLITERAL, regex); Tokens.Add(TokenType.REALLITERAL); regex = new Regex(@"0(x|X)[0-9a-fA-F]+", RegexOptions.Compiled); Patterns.Add(TokenType.HEXINTEGERLITERAL, regex); Tokens.Add(TokenType.HEXINTEGERLITERAL); regex = new Regex(@"\""(\""\""|[^\""])*\""", RegexOptions.Compiled); Patterns.Add(TokenType.STRINGLITERAL, regex); Tokens.Add(TokenType.STRINGLITERAL); regex = new Regex(@"[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()", RegexOptions.Compiled); Patterns.Add(TokenType.FUNCTION, regex); Tokens.Add(TokenType.FUNCTION); regex = new Regex(@"[a-zA-Z_][a-zA-Z0-9_]*(?!\s*\()", RegexOptions.Compiled); Patterns.Add(TokenType.VARIABLE, regex); Tokens.Add(TokenType.VARIABLE); regex = new Regex(@"(?i)pi|e", RegexOptions.Compiled); Patterns.Add(TokenType.CONSTANT, regex); Tokens.Add(TokenType.CONSTANT); regex = new Regex(@"{\s*", RegexOptions.Compiled); Patterns.Add(TokenType.BRACEOPEN, regex); Tokens.Add(TokenType.BRACEOPEN); regex = new Regex(@"\s*}", RegexOptions.Compiled); Patterns.Add(TokenType.BRACECLOSE, regex); Tokens.Add(TokenType.BRACECLOSE); regex = new Regex(@"\(\s*", RegexOptions.Compiled); Patterns.Add(TokenType.BRACKETOPEN, regex); Tokens.Add(TokenType.BRACKETOPEN); regex = new Regex(@"\s*\)", RegexOptions.Compiled); Patterns.Add(TokenType.BRACKETCLOSE, regex); Tokens.Add(TokenType.BRACKETCLOSE); regex = new Regex(@",", RegexOptions.Compiled); Patterns.Add(TokenType.COMMA, regex); Tokens.Add(TokenType.COMMA); regex = new Regex(@"\+\+", RegexOptions.Compiled); Patterns.Add(TokenType.PLUSPLUS, regex); Tokens.Add(TokenType.PLUSPLUS); regex = new Regex(@"--", RegexOptions.Compiled); Patterns.Add(TokenType.MINUSMINUS, regex); Tokens.Add(TokenType.MINUSMINUS); regex = new Regex(@"\|\|", RegexOptions.Compiled); Patterns.Add(TokenType.PIPEPIPE, regex); Tokens.Add(TokenType.PIPEPIPE); regex = new Regex(@"&&", RegexOptions.Compiled); Patterns.Add(TokenType.AMPAMP, regex); Tokens.Add(TokenType.AMPAMP); regex = new Regex(@"&(?!&)", RegexOptions.Compiled); Patterns.Add(TokenType.AMP, regex); Tokens.Add(TokenType.AMP); regex = new Regex(@"\^", RegexOptions.Compiled); Patterns.Add(TokenType.POWER, regex); Tokens.Add(TokenType.POWER); regex = new Regex(@"\+", RegexOptions.Compiled); Patterns.Add(TokenType.PLUS, regex); Tokens.Add(TokenType.PLUS); regex = new Regex(@"-", RegexOptions.Compiled); Patterns.Add(TokenType.MINUS, regex); Tokens.Add(TokenType.MINUS); regex = new Regex(@"=", RegexOptions.Compiled); Patterns.Add(TokenType.EQUAL, regex); Tokens.Add(TokenType.EQUAL); regex = new Regex(@":=", RegexOptions.Compiled); Patterns.Add(TokenType.ASSIGN, regex); Tokens.Add(TokenType.ASSIGN); regex = new Regex(@"!=|<>", RegexOptions.Compiled); Patterns.Add(TokenType.NOTEQUAL, regex); Tokens.Add(TokenType.NOTEQUAL); regex = new Regex(@"!", RegexOptions.Compiled); Patterns.Add(TokenType.NOT, regex); Tokens.Add(TokenType.NOT); regex = new Regex(@"\*", RegexOptions.Compiled); Patterns.Add(TokenType.ASTERIKS, regex); Tokens.Add(TokenType.ASTERIKS); regex = new Regex(@"/", RegexOptions.Compiled); Patterns.Add(TokenType.SLASH, regex); Tokens.Add(TokenType.SLASH); regex = new Regex(@"%", RegexOptions.Compiled); Patterns.Add(TokenType.PERCENT, regex); Tokens.Add(TokenType.PERCENT); regex = new Regex(@"\?", RegexOptions.Compiled); Patterns.Add(TokenType.QUESTIONMARK, regex); Tokens.Add(TokenType.QUESTIONMARK); regex = new Regex(@".", RegexOptions.Compiled); Patterns.Add(TokenType.POINT, regex); Tokens.Add(TokenType.POINT); regex = new Regex(@"<=", RegexOptions.Compiled); Patterns.Add(TokenType.LESSEQUAL, regex); Tokens.Add(TokenType.LESSEQUAL); regex = new Regex(@">=", RegexOptions.Compiled); Patterns.Add(TokenType.GREATEREQUAL, regex); Tokens.Add(TokenType.GREATEREQUAL); regex = new Regex(@"<(?!>)", RegexOptions.Compiled); Patterns.Add(TokenType.LESSTHAN, regex); Tokens.Add(TokenType.LESSTHAN); regex = new Regex(@">", RegexOptions.Compiled); Patterns.Add(TokenType.GREATERTHAN, regex); Tokens.Add(TokenType.GREATERTHAN); regex = new Regex(@":", RegexOptions.Compiled); Patterns.Add(TokenType.COLON, regex); Tokens.Add(TokenType.COLON); regex = new Regex(@"^$", RegexOptions.Compiled); Patterns.Add(TokenType.EOF, regex); Tokens.Add(TokenType.EOF); regex = new Regex(@"\s+", RegexOptions.Compiled); Patterns.Add(TokenType.WHITESPACE, regex); Tokens.Add(TokenType.WHITESPACE); }
public Token Join(Token tok) { if (type == TokenType.Operator && tok.type == TokenType.Number) { return Token.StringToToken(operand + tok.value); } else if (tok.type == TokenType.Operator && type == TokenType.Number) { return Token.StringToToken(tok.operand + value); } return null; }