protected void Reset(LinkedFifoBuffer <TInput> .BufferPosition bufferPosition) { this.buffer = bufferPosition.Buffer; this.tokenStart = this.tokenEnd = this.bufferPosition = bufferPosition; this.tokenSymbol = null; this.state = this.StartStateId; }
public void Push(TInput[] letters, int index, int count) { this.AssertBuffer(); LinkedFifoBuffer <TInput> .Write(ref this.buffer, letters, index, count); this.ProcessData(); }
public void Push(TInput letter) { this.AssertBuffer(); LinkedFifoBuffer <TInput> .Write(ref this.buffer, letter); this.ProcessData(); }
private void AssertBuffer() { if (this.buffer == null) { this.buffer = new LinkedFifoBuffer <TInput>(); this.Reset(this.buffer.HeadPosition); } }
public void Push(IEnumerable <TInput> letters) { this.AssertBuffer(); foreach (var letter in letters) { LinkedFifoBuffer <TInput> .Write(ref this.buffer, letter); } this.ProcessData(); }
private void ProcessData() { TInput value; while (LinkedFifoBuffer <TInput> .TryGetValue(ref this.bufferPosition, out value)) { var newState = this.state; var newSymbol = this.ProcessStateMachine(ref newState, value); if (Dfa <TLetter> .IsEndState(newState)) { if (this.tokenSymbol.HasValue || newState == Dfa <TLetter> .Accept) { var position = this.bufferPosition; this.FlushPendingToken(); if (newState == Dfa <TLetter> .Accept) { this.tokenEnd = this.tokenStart = this.bufferPosition = position; // restore position after EOF letter this.ProcessEof(); } else { continue; } } else { this.HandleLexicalError(false); } } this.state = newState; if (newSymbol.HasValue) { this.tokenEnd = this.bufferPosition; this.tokenSymbol = newSymbol.Value; } } }