public override Token?Lex(TokenBuilder builder) { if (!IncludeNext(builder)) { return(null); } while (IncludeNext(builder)) { ; // DO NOTHING } return(ConstructToken(builder.CollectPreceding())); }
public override Token?Lex(TokenBuilder builder) { try { if (LexNumber(builder) is double value) { return(new NumberToken(value)); } else { return(null); } } catch (ArgumentOutOfRangeException) // Builder ran out of characters unexpectedly { return(null); } }
private double LexDigits(TokenBuilder builder, out int numberOfDigits) { double value = 0; numberOfDigits = 0; while (builder.MoveNext()) { if (AsDigit(builder.Current) is int digit) { value = 10 * value + digit; ++numberOfDigits; } else { break; } } return(value); }
private double?LexNumber(TokenBuilder builder) { double value = LexDigits(builder, out int digits); if (builder.TryMatchCurrent('.')) { double valueAfterDecimal = LexDigits(builder, out int digitsAfterDecimal); for (int i = 0; i < digitsAfterDecimal; ++i) { valueAfterDecimal /= 10; } value += valueAfterDecimal; digits += digitsAfterDecimal; } if (digits == 0) { return(null); } return(value); }
private bool IncludeNext(TokenBuilder builder) => builder.MoveNext() && IsInCharacterClass(builder.Current);
public abstract Token?Lex(TokenBuilder builder);
internal TokenStream(IEnumerator <char> source, IEnumerable <ILexer> lexers) { tokenBuilder = new TokenBuilder(source); this.lexers = lexers; }