protected Token?StartOfRecordDelimiter(ITokenizerStateContext stateContext)
{
var intermediateState = new StringMatchingState(StateHolder.Default, "\n", context =>
{
context.WasLastTokenInRecord = true;
return(Token.CreateValueToken(stateContext.ResetToken()));
});
stateContext.State = intermediateState;
return(null);
}
private Token?TokenFinished(ITokenizerStateContext stateContext)
{
stateContext.State = StateHolder.Default;
return(Token.CreateValueToken(stateContext.ResetToken()));
}
public override Token?Finish(ITokenizerStateContext stateContext)
{
return(Token.CreateValueToken(stateContext.ResetToken()));
}
private Token?TokenFinished(ITokenizerStateContext stateContext)
{
return(Token.CreateValueToken(stateContext.ResetToken()));
}
protected void WithCurrentToken(string tokenValue)
{
A.CallTo(() => TokenizerStateContext.ResetToken()).Returns(tokenValue);
}