Esempio n. 1
0
 public Tokenizer(TextReader input, bool ignoreCase)
 {
     this._stringDfaMatcher = new StringDFAMatcher(ignoreCase);
     this._nfaMatcher       = new NFAMatcher(ignoreCase);
     this._regExpMatcher    = new RegExpMatcher(ignoreCase);
     this._buffer           = new ReaderBuffer(input);
 }
Esempio n. 2
0
 public void Reset(TextReader input)
 {
     //this.buffer.Dispose();
     this._buffer        = new ReaderBuffer(input);
     this._previousToken = null;
     this._lastMatch.Clear();
 }
Esempio n. 3
0
        public override void Match(ReaderBuffer buffer, TokenMatch match)
        {
            TokenPattern res = _automaton.Match(buffer, IgnoreCase);

            if (res != null)
            {
                match.Update(res.Pattern.Length, res);
            }
        }
Esempio n. 4
0
 public override void Match(ReaderBuffer buffer, TokenMatch match)
 {
     for (int i = 0; i < _regExps.Length; i++)
     {
         int length = _regExps[i].Match(buffer);
         if (length > 0)
         {
             match.Update(length, Patterns[i]);
         }
     }
 }
Esempio n. 5
0
 public override int Match(ReaderBuffer buffer)
 {
     if (_matcher == null)
     {
         _matcher = _regExp.Matcher(buffer);
     }
     else
     {
         _matcher.Reset(buffer);
     }
     return(_matcher.MatchFromBeginning() ? _matcher.Length() : 0);
 }
Esempio n. 6
0
        public TokenPattern Match(ReaderBuffer buffer, bool caseInsensitive)
        {
            TokenPattern result = null;
            DFAState     state;
            int          pos = 0;

            var c = buffer.Peek(0);

            if (c < 0)
            {
                return(null);
            }
            if (caseInsensitive)
            {
                c = Char.ToLower((char)c);
            }
            if (c < 128)
            {
                state = _ascii[c];
                if (state == null)
                {
                    return(null);
                }
                else if (state.Value != null)
                {
                    result = state.Value;
                }
                pos++;
            }
            else
            {
                state = _nonAscii;
            }
            while ((c = buffer.Peek(pos)) >= 0)
            {
                state = state.Tree.Find((char)c, caseInsensitive);
                if (state == null)
                {
                    break;
                }
                else if (state.Value != null)
                {
                    result = state.Value;
                }
                pos++;
            }
            return(result);
        }
Esempio n. 7
0
        public int Match(ReaderBuffer buffer, TokenMatch match)
        {
            int      length = 0;
            int      pos    = 1;
            NFAState state;

            // The first step of the match loop has been unrolled and
            // optimized for performance below.
            this._queue.Clear();
            var peekChar = buffer.Peek(0);

            if (0 <= peekChar && peekChar < 128)
            {
                state = this._initialChar[peekChar];
                if (state != null)
                {
                    this._queue.AddLast(state);
                }
            }
            if (peekChar >= 0)
            {
                this._initial.MatchTransitions((char)peekChar, this._queue, true);
            }
            this._queue.MarkEnd();
            peekChar = buffer.Peek(1);

            // The remaining match loop processes all subsequent states
            while (!this._queue.Empty)
            {
                if (this._queue.Marked)
                {
                    pos++;
                    peekChar = buffer.Peek(pos);
                    this._queue.MarkEnd();
                }
                state = this._queue.RemoveFirst();
                if (state.Value != null)
                {
                    match.Update(pos, state.Value);
                }
                if (peekChar >= 0)
                {
                    state.MatchTransitions((char)peekChar, this._queue, false);
                }
            }
            return(length);
        }
Esempio n. 8
0
        public override int Match(ReaderBuffer buffer)
        {
            Match m;

            // Ugly hack since .NET doesn't have a flag for when the
            // end of the input string was encountered...
            buffer.Peek(1024 * 16);
            // Also, there is no API to limit the search to the specified
            // position, so we double-check the index afterwards instead.
            m = _reg.Match(buffer.ToString(), buffer.Position);
            if (m.Success && m.Index == buffer.Position)
            {
                return(m.Length);
            }
            else
            {
                return(0);
            }
        }
Esempio n. 9
0
 public abstract int Match(ReaderBuffer buffer);
Esempio n. 10
0
 public override void Match(ReaderBuffer buffer, TokenMatch match)
 {
     _automaton.Match(buffer, match);
 }
Esempio n. 11
0
 public abstract void Match(ReaderBuffer buffer, TokenMatch match);