public Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character) { if (CharacterDoesNotFit()) { return(Fallback(stateContext, ConsumedCharacters.Append(character))); } _nextCharacter++; if (RemainingCharacters()) { return(null); } stateContext.State = _originalState; return(_onMatch(stateContext)); bool CharacterDoesNotFit() { return(_expectedCharacters[_nextCharacter] != character); } bool RemainingCharacters() { return(_expectedCharacters.Length > _nextCharacter); } }
private Token?Fallback(ITokenizerStateContext stateContext, IEnumerable <char> consumedCharacters) { Token?token = null; foreach (var character in consumedCharacters) { var returnedToken = _originalState.AcceptNextCharacter(stateContext, character); if (CanHoldToken(returnedToken)) { token = returnedToken; } else if (returnedToken.HasValue) { throw new TokenizationException("FallbackState returned more than 1 tokens."); } } stateContext.State = _originalState; return(token); bool CanHoldToken(Token?returnedToken) { return(returnedToken.HasValue && !token.HasValue); } }
public override Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character) { switch (character) { case '"': return(Quote(stateContext)); default: return(NormalOrEscapedCharacter(stateContext, character)); } }
protected Token?StartOfRecordDelimiter(ITokenizerStateContext stateContext) { var intermediateState = new StringMatchingState(StateHolder.Default, "\n", context => { context.WasLastTokenInRecord = true; return(Token.CreateValueToken(stateContext.ResetToken())); }); stateContext.State = intermediateState; return(null); }
public override Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character) { if (character == '"') { return(Quote(stateContext)); } if (character == _configuration.FieldDelimiter) { return(TokenFinished(stateContext)); } if (character == '\r') { return(StartOfRecordDelimiter(stateContext)); } return(NormalCharacter()); }
public void SetUpBase() { TokenizerStateContext = A.Fake <ITokenizerStateContext>(); }
private Token?NormalOrEscapedCharacter(ITokenizerStateContext stateContext, char character) { stateContext.AppendCharacter(character); return(null); }
private Token?Quote(ITokenizerStateContext stateContext) { stateContext.State = StateHolder.EscapedWithOneQuote; return(null); }
public override Token?Finish(ITokenizerStateContext stateContext) { throw new TokenizationException("Escaped field not closed. Unexpected end of input."); }
private Token?Quote(ITokenizerStateContext stateContext) { stateContext.State = StateHolder.Escaped; stateContext.AppendCharacter('"'); return(null); }
private Token?TokenFinished(ITokenizerStateContext stateContext) { stateContext.State = StateHolder.Default; return(Token.CreateValueToken(stateContext.ResetToken())); }
public override Token?Finish(ITokenizerStateContext stateContext) { return(Token.CreateValueToken(stateContext.ResetToken())); }
private Token?TokenFinished(ITokenizerStateContext stateContext) { return(Token.CreateValueToken(stateContext.ResetToken())); }
public Token?Finish(ITokenizerStateContext stateContext) { return(Fallback(stateContext, ConsumedCharacters)); }
private Token?Quote(ITokenizerStateContext stateContext) { stateContext.State = StateHolder.Escaped; // TODO Throw error, if quote not at start return(null); }
public abstract Token?Finish(ITokenizerStateContext stateContext);
public abstract Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character);