protected override IToken ExtractToken(string str) { string value = null; string type = null; foreach (var kvp in Recognizers) { IRecognizer recognizer = kvp.Value; string v; if (recognizer.Match(str, out v)) { if (value == null || Util.Util.StringLength(v) > Util.Util.StringLength(value)) { value = v; type = kvp.Key; } } } if (type != null) { return(new CommonToken(type, value, this.Line)); } return(null); }
protected override IToken ExtractToken(string str) { if (!this.StateStack.Any()) { throw new InvalidOperationException("You must set a starting state before lexing."); } string value = null; string type = null; object action = null; var state = this.States[this.StateStack.Peek()]; foreach (var kvp in state.Recognizers) { IRecognizer recognizer = kvp.Value; string v; if (recognizer.Match(str, out v)) { if (value == null || Util.Util.StringLength(v) > Util.Util.StringLength(value)) { value = v; type = kvp.Key; action = state.Actions[type]; } } } if (type != null) { string actionString = action as string; if (actionString != null) { // enter new state this.StateStack.Push(actionString); } else if ((int)action == POP_STATE) { this.StateStack.Pop(); } return(new CommonToken(type, value, this.Line)); } return(null); }