public IToken Produce()
{
var context = this.Context;
var text = context.Text;
var length = text.Length;
var c = text[context.Index];
if (LexingHelper.IsLatinLetter(c) || c == '_')
{
var initialIndex = context.Index;
var index = initialIndex + 1;
var column = context.Column + 1;
while (true)
{
if (index == length)
{
break;
}
c = text[index];
if (
LexingHelper.IsInlineWhiteSpaceOrCaretControl(c) ||
LexingHelper.IsStandardPunctuationChar(c))
{
break;
}
if (c == '_' || LexingHelper.IsLatinLetter(c) || LexingHelper.IsDigit(c))
{
index++;
column++;
continue;
}
return(null);
}
var delta = index - initialIndex;
var str = text.Substring(initialIndex, delta);
context.Advance(delta, 0, column);
return(new TextToken(
WordTextClass.Instance,
NoneTextDecoration.Instance,
str,
new Position(context.Line, column), delta));
}
return(null);
}
public PunctuationToken(
char c,
Position position,
int consumedLength)
: base(position, consumedLength)
{
if (!LexingHelper.IsStandardPunctuationChar(c))
{
throw new ArgumentOutOfRangeException(nameof(c));
}
this.Value = c;
}
public ExactPunctuationNode(
char c,
Action <ActionNode, IToken, IResultAccumulator> action,
INodeFamily family,
string name)
: base(action, family, name)
{
if (!LexingHelper.IsStandardPunctuationChar(c))
{
throw new ArgumentOutOfRangeException(nameof(c));
}
this.Value = c;
}
