protected override IToken ExtractToken(string str)
{
string value = null;
string type = null;
foreach (var kvp in Recognizers)
{
IRecognizer recognizer = kvp.Value;
string v;
if (recognizer.Match(str, out v))
{
if (value == null || Util.Util.StringLength(v) > Util.Util.StringLength(value))
{
value = v;
type = kvp.Key;
}
}
}
if (type != null)
{
return(new CommonToken(type, value, this.Line));
}
return(null);
}
protected override IToken ExtractToken(string str)
{
if (!this.StateStack.Any())
{
throw new InvalidOperationException("You must set a starting state before lexing.");
}
string value = null;
string type = null;
object action = null;
var state = this.States[this.StateStack.Peek()];
foreach (var kvp in state.Recognizers)
{
IRecognizer recognizer = kvp.Value;
string v;
if (recognizer.Match(str, out v))
{
if (value == null || Util.Util.StringLength(v) > Util.Util.StringLength(value))
{
value = v;
type = kvp.Key;
action = state.Actions[type];
}
}
}
if (type != null)
{
string actionString = action as string;
if (actionString != null)
{ // enter new state
this.StateStack.Push(actionString);
}
else if ((int)action == POP_STATE)
{
this.StateStack.Pop();
}
return(new CommonToken(type, value, this.Line));
}
return(null);
}
