public Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character)
{
if (CharacterDoesNotFit())
{
return(Fallback(stateContext, ConsumedCharacters.Append(character)));
}
_nextCharacter++;
if (RemainingCharacters())
{
return(null);
}
stateContext.State = _originalState;
return(_onMatch(stateContext));
bool CharacterDoesNotFit()
{
return(_expectedCharacters[_nextCharacter] != character);
}
bool RemainingCharacters()
{
return(_expectedCharacters.Length > _nextCharacter);
}
}
private Token?Fallback(ITokenizerStateContext stateContext, IEnumerable <char> consumedCharacters)
{
Token?token = null;
foreach (var character in consumedCharacters)
{
var returnedToken = _originalState.AcceptNextCharacter(stateContext, character);
if (CanHoldToken(returnedToken))
{
token = returnedToken;
}
else if (returnedToken.HasValue)
{
throw new TokenizationException("FallbackState returned more than 1 tokens.");
}
}
stateContext.State = _originalState;
return(token);
bool CanHoldToken(Token?returnedToken)
{
return(returnedToken.HasValue && !token.HasValue);
}
}
public override Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character)
{
switch (character)
{
case '"':
return(Quote(stateContext));
default:
return(NormalOrEscapedCharacter(stateContext, character));
}
}
protected Token?StartOfRecordDelimiter(ITokenizerStateContext stateContext)
{
var intermediateState = new StringMatchingState(StateHolder.Default, "\n", context =>
{
context.WasLastTokenInRecord = true;
return(Token.CreateValueToken(stateContext.ResetToken()));
});
stateContext.State = intermediateState;
return(null);
}
public override Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character)
{
if (character == '"')
{
return(Quote(stateContext));
}
if (character == _configuration.FieldDelimiter)
{
return(TokenFinished(stateContext));
}
if (character == '\r')
{
return(StartOfRecordDelimiter(stateContext));
}
return(NormalCharacter());
}
public void SetUpBase()
{
TokenizerStateContext = A.Fake <ITokenizerStateContext>();
}
private Token?NormalOrEscapedCharacter(ITokenizerStateContext stateContext, char character)
{
stateContext.AppendCharacter(character);
return(null);
}
private Token?Quote(ITokenizerStateContext stateContext)
{
stateContext.State = StateHolder.EscapedWithOneQuote;
return(null);
}
public override Token?Finish(ITokenizerStateContext stateContext)
{
throw new TokenizationException("Escaped field not closed. Unexpected end of input.");
}
private Token?Quote(ITokenizerStateContext stateContext)
{
stateContext.State = StateHolder.Escaped;
stateContext.AppendCharacter('"');
return(null);
}
private Token?TokenFinished(ITokenizerStateContext stateContext)
{
stateContext.State = StateHolder.Default;
return(Token.CreateValueToken(stateContext.ResetToken()));
}
public override Token?Finish(ITokenizerStateContext stateContext)
{
return(Token.CreateValueToken(stateContext.ResetToken()));
}
private Token?TokenFinished(ITokenizerStateContext stateContext)
{
return(Token.CreateValueToken(stateContext.ResetToken()));
}
public Token?Finish(ITokenizerStateContext stateContext)
{
return(Fallback(stateContext, ConsumedCharacters));
}
private Token?Quote(ITokenizerStateContext stateContext)
{
stateContext.State = StateHolder.Escaped;
// TODO Throw error, if quote not at start
return(null);
}
public abstract Token?Finish(ITokenizerStateContext stateContext);
public abstract Token?AcceptNextCharacter(ITokenizerStateContext stateContext, char character);
