/// <summary> /// Basic error recovery that adds a samicolon if it is possible. /// </summary> /// <param name="lexerState">Current parser state.</param> /// <returns><c>True</c> is semicolon is expected, <c>False</c> otherwise.</returns> public bool TryRecover(ILexerState lexerState) { if (lexerState.IsExpectedToken((int)Tokens.T_SEMI)) { lexerState.SetToken(0, new CompleteToken(Tokens.T_SEMI, new SemanticValueType(), new Span(lexerState.PreviousToken.TokenPosition.End, 0), ";")); return(true); } return(false); }
public IEnumerable <IToken> Tokenize(string expression) { states_ = new Stack <ILexerState>(); currentState_ = new LiteralLexerState(new CommonState()); characters_ = expression.GetEnumerator(); while (characters_.MoveNext()) { IToken token = currentState_.Handle(this); if (token != null) { yield return(token); } } }
private Token MakeToken(ILexerState state) => state.Token.With(v => v.TypeMatch() .With(Case.Is <NumberLexerState>(), _ => (Token) new IntegerToken() { Value = int.Parse(v.TokenValue.Value) }) .With(Case.Is <SymbolLexerState>(), _ => v.TokenValue.Match() .With(_parsers.KeywordParser.Is, _parsers.KeywordParser.Make) .Else(id => new IdentifierToken { Identifier = id }) .Do()) .Else(_ => v.TokenValue.Match() .With(_parsers.SymbolParser.Is, _parsers.SymbolParser.Make) .With(_parsers.OperatorParser.Is, _parsers.OperatorParser.Make) .Do() ) .Do() );
public ILexerState CreateTailToken(IForwardReadSequence <char> sequence, ILexerState state, IParserSet parsers) => new InitialLexerState(sequence, (LexerStateBase)state, parsers);
/// <summary> /// Always fails to recover. /// </summary> /// <param name="lexerState">Current parser state.</param> /// <returns>Always <c>False</c>.</returns> public bool TryRecover(ILexerState lexerState) => false;
public void EndState() { currentState_ = states_.Pop(); }
public void ToState(ILexerState state) { states_.Push(currentState_); currentState_ = state; }