public override Token Match(Lexer lexer) { bool isKeyword = false; string word = string.Empty; using (var la = new LookaheadFrame(lexer)) { var c = lexer.NextChar(); if (IsIdentifierStartChar(c)) { var tok = new Token() { Line = lexer.Line, Col = lexer.Col, Sym = Sym.Id }; var sb = new StringBuilder(); sb.Append(c); while (IsIdentifierPartChar(lexer.Peek())) { sb.Append(lexer.NextChar()); } //Since the id rule tries to match before any //reserved-word ones, we need to make sure this id isn't //actually a keyword word = sb.ToString(); isKeyword = lexer.KeywordTable.ContainsKey(word); if (!isKeyword) { la.Commit(); tok.Value = word; return tok; } } } //If it is a kw, invoke that rule instead if (isKeyword) { return lexer.KeywordTable[word].Match(lexer); } return null; }
public override Token Match(Lexer lexer) { var rem = lexer.RemainingInput; if (lexer.CanRead) { string value = null; var match = singleCharRe.Match(rem); if (match.Success) { value = match.Value; } else if ((match = escapeCharRe.Match(rem)).Success) { value = match.Value; } else if ((match = hexEscapeRe.Match(rem)).Success) { value = match.Value; } else if ((match = unicEscapeRe.Match(rem)).Success) { value = match.Value; } if (null != value) { var tok = new Token() { Line = lexer.Line, Col = lexer.Col }; tok.Sym = Sym.CharLiteral; tok.Value = value.Substring(1, value.Length - 2); ; lexer.Advance(value.Length); return tok; } } return null; }
private string FormatError(string msg, Lexer lexer) { var tok = lexer.LastOne; if (null == tok) return msg; return string.Format("Line {0}, col {1}: {2}", tok.Line, tok.Col, msg); }
//TODO: 'Verbatim' string handling (e.g. @"...") public override Token Match(Lexer lexer) { if (lexer.CanRead) { var match = simpleLit.Match(lexer.RemainingInput); if (match.Success) { var tok = new Token() { Line = lexer.Line, Col = lexer.Col }; tok.Sym = Sym.StringLiteral; tok.Value = match.Value.Substring(1, match.Value.Length - 2); lexer.Advance(match.Value.Length); return tok; } } return null; }
public override Token Match(Lexer lexer) { var tok = new Token() { Line = lexer.Line, Col = lexer.Col }; string rem = lexer.RemainingInput; bool isMatch = false; Match match = hexIntLit.Match(rem); if (match.Success) { tok.Value = match.Value; tok.Sym = Sym.HexIntLiteral; isMatch = true; } foreach (var re in realRes) { match = re.Match(rem); if (match.Success) { tok.Value = match.Value; tok.Sym = Sym.RealLiteral; isMatch = true; break; } } if (!isMatch) { match = decIntLit.Match(rem); if (match.Success) { tok.Value = match.Value; tok.Sym = Sym.IntLiteral; isMatch = true; } } if (isMatch) { lexer.Advance(tok.Value.Length); return tok; } return null; }
public void Error(string msg, Lexer lexer) { errors.Add(FormatError(msg, lexer)); }
public void TestLexicalMatch1() { var lexer = new Lexer("foo bar"); var rule = new LexicalRule("foo", Sym.KwClass); Assert.IsNotNull(rule.Match(lexer)); }