public override Token?Lex(TokenBuilder builder)
{
if (!IncludeNext(builder))
{
return(null);
}
while (IncludeNext(builder))
{
; // DO NOTHING
}
return(ConstructToken(builder.CollectPreceding()));
}
public override Token?Lex(TokenBuilder builder)
{
try
{
if (LexNumber(builder) is double value)
{
return(new NumberToken(value));
}
else
{
return(null);
}
}
catch (ArgumentOutOfRangeException) // Builder ran out of characters unexpectedly
{
return(null);
}
}
private double LexDigits(TokenBuilder builder, out int numberOfDigits)
{
double value = 0;
numberOfDigits = 0;
while (builder.MoveNext())
{
if (AsDigit(builder.Current) is int digit)
{
value = 10 * value + digit;
++numberOfDigits;
}
else
{
break;
}
}
return(value);
}
private double?LexNumber(TokenBuilder builder)
{
double value = LexDigits(builder, out int digits);
if (builder.TryMatchCurrent('.'))
{
double valueAfterDecimal = LexDigits(builder, out int digitsAfterDecimal);
for (int i = 0; i < digitsAfterDecimal; ++i)
{
valueAfterDecimal /= 10;
}
value += valueAfterDecimal;
digits += digitsAfterDecimal;
}
if (digits == 0)
{
return(null);
}
return(value);
}
private bool IncludeNext(TokenBuilder builder) => builder.MoveNext() && IsInCharacterClass(builder.Current);
public abstract Token?Lex(TokenBuilder builder);
internal TokenStream(IEnumerator <char> source, IEnumerable <ILexer> lexers)
{
tokenBuilder = new TokenBuilder(source);
this.lexers = lexers;
}