protected virtual Token NewToken(TokenPattern pattern, string image, int line, int column) { return(new Token(pattern, image, line, column)); }
public void AddTextMatch(string str, bool ignoreCase, TokenPattern value) { NFAState state; char ch = str[0]; if (ch < 128 && !ignoreCase) { state = _initialChar[ch]; if (state == null) { state = _initialChar[ch] = new NFAState(); } } else { state = _initial.AddOut(ch, ignoreCase, null); } for (int i = 1; i < str.Length; i++) { state = state.AddOut(str[i], ignoreCase, null); } state.Value = value; }
public override void Match(ReaderBuffer buffer, TokenMatch match) { TokenPattern res = _automaton.Match(buffer, IgnoreCase); if (res != null) { match.Update(res.Pattern.Length, res); } }
public TokenPattern Match(ReaderBuffer buffer, bool caseInsensitive) { TokenPattern result = null; DFAState state; int pos = 0; var c = buffer.Peek(0); if (c < 0) { return(null); } if (caseInsensitive) { c = Char.ToLower((char)c); } if (c < 128) { state = _ascii[c]; if (state == null) { return(null); } else if (state.Value != null) { result = state.Value; } pos++; } else { state = _nonAscii; } while ((c = buffer.Peek(pos)) >= 0) { state = state.Tree.Find((char)c, caseInsensitive); if (state == null) { break; } else if (state.Value != null) { result = state.Value; } pos++; } return(result); }
public override void AddPattern(TokenPattern pattern) { if (pattern.Type == TokenPattern.PatternType.STRING) { _automaton.AddTextMatch(pattern.Pattern, IgnoreCase, pattern); } else { _automaton.AddRegExpMatch(pattern.Pattern, IgnoreCase, pattern); } base.AddPattern(pattern); }
public Token(TokenPattern pattern, string image, int line, int col) { this._pattern = pattern; this._image = image; this._startLine = line; this._startColumn = col; this._endLine = line; this._endColumn = col + image.Length - 1; for (int pos = 0; image.IndexOf('\n', pos) >= 0;) { pos = image.IndexOf('\n', pos) + 1; this._endLine++; _endColumn = image.Length - pos; } }
public override void AddPattern(TokenPattern pattern) { REHandler re; try { re = new GrammaticaRE(pattern.Pattern, IgnoreCase); pattern.DebugInfo = "Grammatica regexp\n" + re; } catch (Exception) { re = new SystemRE(pattern.Pattern, IgnoreCase); pattern.DebugInfo = "native .NET regexp"; } Array.Resize(ref _regExps, _regExps.Length + 1); _regExps[_regExps.Length - 1] = re; base.AddPattern(pattern); }
public void AddPattern(TokenPattern pattern) { switch (pattern.Type) { case TokenPattern.PatternType.STRING: try { _stringDfaMatcher.AddPattern(pattern); } catch (Exception e) { throw new ParserCreationException( ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "error adding string token: " + e.Message); } break; case TokenPattern.PatternType.REGEXP: try { _nfaMatcher.AddPattern(pattern); } catch (Exception) { try { _regExpMatcher.AddPattern(pattern); } catch (Exception e) { throw new ParserCreationException( ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "regular expression contains error(s): " + e.Message); } } break; default: throw new ParserCreationException( ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "pattern type " + pattern.Type + " is undefined"); } }
public void AddMatch(string str, bool caseInsensitive, TokenPattern value) { DFAState state; char c = str[0]; int start = 0; if (caseInsensitive) { c = Char.ToLower(c); } if (c < 128) { state = _ascii[c]; if (state == null) { state = _ascii[c] = new DFAState(); } start++; } else { state = _nonAscii; } for (int i = start; i < str.Length; i++) { var next = state.Tree.Find(str[i], caseInsensitive); if (next == null) { next = new DFAState(); state.Tree.Add(str[i], caseInsensitive, next); } state = next; } state.Value = value; }
public void AddRegExpMatch(string pattern, bool ignoreCase, TokenPattern value) { TokenRegExpParser parser = new TokenRegExpParser(pattern, ignoreCase); string debug = "DFA regexp; " + parser.GetDebugInfo(); var isAscii = parser.Start.IsAsciiOutgoing(); for (int i = 0; isAscii && i < 128; i++) { bool match = false; for (int j = 0; j < parser.Start.Outgoing.Length; j++) { if (parser.Start.Outgoing[j].Match((char)i)) { if (match) { isAscii = false; break; } match = true; } } if (match && _initialChar[i] != null) { isAscii = false; } } if (parser.Start.Incoming.Length > 0) { _initial.AddOut(new NFAEpsilonTransition(parser.Start)); debug += ", uses initial epsilon"; } else if (isAscii && !ignoreCase) { for (int i = 0; isAscii && i < 128; i++) { for (int j = 0; j < parser.Start.Outgoing.Length; j++) { if (parser.Start.Outgoing[j].Match((char)i)) { _initialChar[i] = parser.Start.Outgoing[j].State; } } } debug += ", uses ASCII lookup"; } else { parser.Start.MergeInto(_initial); debug += ", uses initial state"; } parser.End.Value = value; value.DebugInfo = debug; }
public override void AddPattern(TokenPattern pattern) { _automaton.AddMatch(pattern.Pattern, IgnoreCase, pattern); base.AddPattern(pattern); }
public virtual void AddPattern(TokenPattern pattern) { Array.Resize(ref Patterns, Patterns.Length + 1); Patterns[Patterns.Length - 1] = pattern; }